{channelList.map((item) => {
- const providerData = aiproxyIdMap[item.type];
+ const providerData = aiproxyIdMap[item.type] || {
+ label: channelProviders[item.type]?.name || 'Invalid provider',
+ provider: 'Other'
+ };
const provider = getModelProvider(providerData?.provider);
return (
@@ -119,14 +132,10 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
| {item.id} |
{item.name} |
- {providerData ? (
-
-
- {t(providerData?.label as any)}
-
- ) : (
- 'Invalid provider'
- )}
+
+
+ {t(providerData?.label as any)}
+
|
Date: Fri, 21 Mar 2025 16:21:46 +0800
Subject: [PATCH 12/35] fix: member count (#4269)
---
.../pageComponents/account/team/context.tsx | 27 +++++++++++++++----
projects/app/src/web/support/user/team/api.ts | 2 ++
2 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/projects/app/src/pageComponents/account/team/context.tsx b/projects/app/src/pageComponents/account/team/context.tsx
index d2e264a5f..509eda64f 100644
--- a/projects/app/src/pageComponents/account/team/context.tsx
+++ b/projects/app/src/pageComponents/account/team/context.tsx
@@ -1,8 +1,13 @@
-import React, { ReactNode, useState } from 'react';
+import React, { ReactNode, useCallback, useState } from 'react';
import { createContext } from 'use-context-selector';
import type { EditTeamFormDataType } from './EditInfoModal';
import dynamic from 'next/dynamic';
-import { getTeamList, getTeamMembers, putSwitchTeam } from '@/web/support/user/team/api';
+import {
+ getTeamList,
+ getTeamMemberCount,
+ getTeamMembers,
+ putSwitchTeam
+} from '@/web/support/user/team/api';
import { TeamMemberStatusEnum } from '@fastgpt/global/support/user/team/constant';
import { useUserStore } from '@/web/support/user/useUserStore';
import type { TeamTmbItemType, TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
@@ -84,12 +89,19 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
refreshDeps: [userInfo?.team?.teamId]
});
+ const { data: teamMemberCountData, refresh: refetchTeamMemberCount } = useRequest2(
+ getTeamMemberCount,
+ {
+ manual: false,
+ refreshDeps: [userInfo?.team?.teamId]
+ }
+ );
+
// member action
const {
data: members = [],
isLoading: loadingMembers,
- refreshList: refetchMembers,
- total: memberTotal,
+ refreshList: refetchMemberList,
ScrollData: MemberScrollData
} = useScrollPagination(getTeamMembers, {
pageSize: 1000,
@@ -98,6 +110,11 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
}
});
+ const refetchMembers = useCallback(() => {
+ refetchTeamMemberCount();
+ refetchMemberList();
+ }, [refetchTeamMemberCount, refetchMemberList]);
+
const { runAsync: onSwitchTeam, loading: isSwitchingTeam } = useRequest2(
async (teamId: string) => {
await putSwitchTeam(teamId);
@@ -135,7 +152,7 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
refetchMembers,
groups,
refetchGroups,
- teamSize: memberTotal,
+ teamSize: teamMemberCountData?.count || 0,
MemberScrollData
};
diff --git a/projects/app/src/web/support/user/team/api.ts b/projects/app/src/web/support/user/team/api.ts
index 62b9bd5b7..8a2086e55 100644
--- a/projects/app/src/web/support/user/team/api.ts
+++ b/projects/app/src/web/support/user/team/api.ts
@@ -36,6 +36,8 @@ export const putSwitchTeam = (teamId: string) =>
/* --------------- team member ---------------- */
export const getTeamMembers = (props: PaginationProps<{ withLeaved?: boolean }>) =>
GET>(`/proApi/support/user/team/member/list`, props);
+export const getTeamMemberCount = () =>
+ GET<{ count: number }>(`/proApi/support/user/team/member/count`);
// export const postInviteTeamMember = (data: InviteMemberProps) =>
// POST(`/proApi/support/user/team/member/invite`, data);
From e812ad6e848a005f07a237882ae04857a8bc63d6 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Fri, 21 Mar 2025 16:44:25 +0800
Subject: [PATCH 13/35] feat: chunk index independent config (#4271)
* sync collection
* remove lock
* feat: chunk index independent config
* feat: add max chunksize to split chunk function
* remove log
* update doc
* remove
* remove log
---
.../zh-cn/docs/development/openapi/dataset.md | 35 +---
.../zh-cn/docs/development/upgrading/492.md | 11 +-
packages/global/common/string/textSplitter.ts | 74 ++++---
packages/global/core/dataset/api.d.ts | 13 +-
packages/global/core/dataset/constants.ts | 10 +
packages/global/core/dataset/controller.d.ts | 1 +
.../global/core/dataset/training/type.d.ts | 2 +
.../global/core/dataset/training/utils.ts | 136 +++++++++++++
packages/global/core/dataset/type.d.ts | 9 +-
packages/global/core/dataset/utils.ts | 21 --
.../core/dataset/collection/controller.ts | 47 +++--
.../service/core/dataset/collection/schema.ts | 17 +-
packages/service/core/dataset/read.ts | 4 +-
.../core/dataset/training/controller.ts | 41 ++--
.../service/core/dataset/training/schema.ts | 1 +
.../common/MyModal/EditFolderModal.tsx | 2 +-
.../components/common/Radio/RadioGroup.tsx | 67 ++++++
packages/web/i18n/en/common.json | 1 -
packages/web/i18n/en/dataset.json | 7 +-
packages/web/i18n/zh-CN/common.json | 1 -
packages/web/i18n/zh-CN/dataset.json | 7 +-
packages/web/i18n/zh-Hant/common.json | 1 -
packages/web/i18n/zh-Hant/dataset.json | 7 +-
.../common/Modal/EditResourceModal.tsx | 2 +-
.../src/components/support/apikey/Table.tsx | 2 +-
.../account/team/EditInfoModal.tsx | 2 +-
.../app/detail/Publish/Link/index.tsx | 2 +-
.../detail/Publish/components/BasicInfo.tsx | 2 +-
.../nodes/NodeExtract/ExtractFieldModal.tsx | 2 +-
.../Flow/nodes/render/NodeCard.tsx | 2 +-
.../app/list/TemplateMarketModal.tsx | 2 +-
.../dataset/EditFolderModal.tsx | 2 +-
.../dataset/detail/Import/Context.tsx | 83 +++++---
.../Import/commonProgress/DataProcess.tsx | 191 +++++++++++-------
.../Import/commonProgress/PreviewData.tsx | 14 +-
.../detail/Import/commonProgress/Upload.tsx | 12 +-
.../Import/components/PreviewChunks.tsx | 102 ----------
.../detail/Import/diffSource/ReTraining.tsx | 17 +-
projects/app/src/pages/account/info/index.tsx | 2 +-
.../collection/create/reTrainingCollection.ts | 3 +-
.../pages/api/core/dataset/data/insertData.ts | 13 +-
.../api/core/dataset/file/getPreviewChunks.ts | 116 +++++++----
.../service/core/dataset/data/controller.ts | 102 ++++++++--
projects/app/src/service/events/generateQA.ts | 27 ++-
.../app/src/service/events/generateVector.ts | 2 +-
.../app/src/web/core/dataset/constants.ts | 6 +-
projects/app/src/web/core/dataset/type.d.ts | 4 +-
47 files changed, 784 insertions(+), 443 deletions(-)
create mode 100644 packages/global/core/dataset/training/utils.ts
create mode 100644 packages/web/components/common/Radio/RadioGroup.tsx
delete mode 100644 projects/app/src/pageComponents/dataset/detail/Import/components/PreviewChunks.tsx
diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md
index d43b2026d..e8cde92fc 100644
--- a/docSite/content/zh-cn/docs/development/openapi/dataset.md
+++ b/docSite/content/zh-cn/docs/development/openapi/dataset.md
@@ -11,8 +11,6 @@ weight: 853
| --------------------- | --------------------- |
|  |  |
-
-
## 创建训练订单
{{< tabs tabTotal="2" >}}
@@ -289,7 +287,7 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete?
## 集合
-### 通用创建参数说明
+### 通用创建参数说明(必看)
**入参**
@@ -300,8 +298,11 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete?
| trainingType | 数据处理方式。chunk: 按文本长度进行分割;qa: 问答对提取 | ✅ |
| autoIndexes | 是否自动生成索引(仅商业版支持) | |
| imageIndex | 是否自动生成图片索引(仅商业版支持) | |
-| chunkSize | 预估块大小 | |
-| chunkSplitter | 自定义最高优先分割符号 | |
+| chunkSettingMode | 分块参数模式。auto: 系统默认参数; custom: 手动指定参数 | |
+| chunkSplitMode | 分块拆分模式。size: 按长度拆分; char: 按字符拆分。chunkSettingMode=auto时不生效。 | |
+| chunkSize | 分块大小,默认 1500。chunkSettingMode=auto时不生效。 | |
+| indexSize | 索引大小,默认 512,必须小于索引模型最大token。chunkSettingMode=auto时不生效。 | |
+| chunkSplitter | 自定义最高优先分割符号,除非超出文件处理最大上下文,否则不会进行进一步拆分。chunkSettingMode=auto时不生效。 | |
| qaPrompt | qa拆分提示词 | |
| tags | 集合标签(字符串数组) | |
| createTime | 文件创建时间(Date / String) | |
@@ -389,9 +390,8 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/collectio
"name":"测试训练",
"trainingType": "qa",
- "chunkSize":8000,
- "chunkSplitter":"",
- "qaPrompt":"11",
+ "chunkSettingMode": "auto",
+ "qaPrompt":"",
"metadata":{}
}'
@@ -409,10 +409,6 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/collectio
- parentId: 父级ID,不填则默认为根目录
- name: 集合名称(必填)
- metadata: 元数据(暂时没啥用)
-- trainingType: 训练模式(必填)
-- chunkSize: 每个 chunk 的长度(可选). chunk模式:100~3000; qa模式: 4000~模型最大token(16k模型通常建议不超过10000)
-- chunkSplitter: 自定义最高优先分割符号(可选)
-- qaPrompt: qa拆分自定义提示词(可选)
{{% /alert %}}
{{< /markdownify >}}
@@ -462,8 +458,7 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/collectio
"parentId": null,
"trainingType": "chunk",
- "chunkSize":512,
- "chunkSplitter":"",
+ "chunkSettingMode": "auto",
"qaPrompt":"",
"metadata":{
@@ -483,10 +478,6 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/collectio
- datasetId: 知识库的ID(必填)
- parentId: 父级ID,不填则默认为根目录
- metadata.webPageSelector: 网页选择器,用于指定网页中的哪个元素作为文本(可选)
-- trainingType:训练模式(必填)
-- chunkSize: 每个 chunk 的长度(可选). chunk模式:100~3000; qa模式: 4000~模型最大token(16k模型通常建议不超过10000)
-- chunkSplitter: 自定义最高优先分割符号(可选)
-- qaPrompt: qa拆分自定义提示词(可选)
{{% /alert %}}
{{< /markdownify >}}
@@ -545,13 +536,7 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/collectio
{{% alert icon=" " context="success" %}}
- file: 文件
-- data: 知识库相关信息(json序列化后传入)
- - datasetId: 知识库的ID(必填)
- - parentId: 父级ID,不填则默认为根目录
- - trainingType:训练模式(必填)
- - chunkSize: 每个 chunk 的长度(可选). chunk模式:100~3000; qa模式: 4000~模型最大token(16k模型通常建议不超过10000)
- - chunkSplitter: 自定义最高优先分割符号(可选)
- - qaPrompt: qa拆分自定义提示词(可选)
+- data: 知识库相关信息(json序列化后传入),参数说明见上方“通用创建参数说明”
{{% /alert %}}
{{< /markdownify >}}
diff --git a/docSite/content/zh-cn/docs/development/upgrading/492.md b/docSite/content/zh-cn/docs/development/upgrading/492.md
index ec006a73b..73d156129 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
@@ -7,12 +7,17 @@ toc: true
weight: 799
---
+## 重要提示
+
+- 知识库导入数据 API 变更,增加`chunkSettingMode`,`chunkSplitMode`,`indexSize`可选参数,具体可参考 [知识库导入数据 API](/docs/development/openapi/dataset) 文档。
+
## 🚀 新增内容
-1. 知识库分块增加自定义分隔符预设值,同时支持自定义换行符分割。
-2. 外部变量改名:自定义变量。 并且支持在测试时调试,在分享链接中,该变量直接隐藏。
-3. 集合同步时,支持同步修改标题。
+1. 知识库分块优化:支持单独配置分块大小和索引大小,允许进行超大分块,以更大的输入 Tokens 换取完整分块。
+2. 知识库分块增加自定义分隔符预设值,同时支持自定义换行符分割。
+3. 外部变量改名:自定义变量。 并且支持在测试时调试,在分享链接中,该变量直接隐藏。
+4. 集合同步时,支持同步修改标题。
## ⚙️ 优化
diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts
index 8c56029dd..c0b335af7 100644
--- a/packages/global/common/string/textSplitter.ts
+++ b/packages/global/common/string/textSplitter.ts
@@ -1,15 +1,17 @@
+import { defaultMaxChunkSize } from '../../core/dataset/training/utils';
import { getErrText } from '../error/utils';
export const CUSTOM_SPLIT_SIGN = '-----CUSTOM_SPLIT_SIGN-----';
type SplitProps = {
text: string;
- chunkLen: number;
+ chunkSize: number;
+ maxSize?: number;
overlapRatio?: number;
customReg?: string[];
};
-export type TextSplitProps = Omit & {
- chunkLen?: number;
+export type TextSplitProps = Omit & {
+ chunkSize?: number;
};
type SplitResponse = {
@@ -55,7 +57,7 @@ const strIsMdTable = (str: string) => {
return true;
};
const markdownTableSplit = (props: SplitProps): SplitResponse => {
- let { text = '', chunkLen } = props;
+ let { text = '', chunkSize } = props;
const splitText2Lines = text.split('\n');
const header = splitText2Lines[0];
const headerSize = header.split('|').length - 2;
@@ -71,7 +73,7 @@ ${mdSplitString}
`;
for (let i = 2; i < splitText2Lines.length; i++) {
- if (chunk.length + splitText2Lines[i].length > chunkLen * 1.2) {
+ if (chunk.length + splitText2Lines[i].length > chunkSize * 1.2) {
chunks.push(chunk);
chunk = `${header}
${mdSplitString}
@@ -98,11 +100,17 @@ ${mdSplitString}
5. 标点分割:重叠
*/
const commonSplit = (props: SplitProps): SplitResponse => {
- let { text = '', chunkLen, overlapRatio = 0.15, customReg = [] } = props;
+ let {
+ text = '',
+ chunkSize,
+ maxSize = defaultMaxChunkSize,
+ overlapRatio = 0.15,
+ customReg = []
+ } = props;
const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
- const overlapLen = Math.round(chunkLen * overlapRatio);
+ const overlapLen = Math.round(chunkSize * overlapRatio);
// replace code block all \n to codeBlockMarker
text = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, function (match) {
@@ -118,24 +126,24 @@ const commonSplit = (props: SplitProps): SplitResponse => {
const stepReges: { reg: RegExp | string; maxLen: number }[] = [
...customReg.map((text) => ({
reg: text.replaceAll('\\n', '\n'),
- maxLen: chunkLen * 1.4
+ maxLen: chunkSize
})),
- { reg: /^(#\s[^\n]+\n)/gm, maxLen: chunkLen * 1.2 },
- { reg: /^(##\s[^\n]+\n)/gm, maxLen: chunkLen * 1.4 },
- { reg: /^(###\s[^\n]+\n)/gm, maxLen: chunkLen * 1.6 },
- { reg: /^(####\s[^\n]+\n)/gm, maxLen: chunkLen * 1.8 },
- { reg: /^(#####\s[^\n]+\n)/gm, maxLen: chunkLen * 1.8 },
+ { reg: /^(#\s[^\n]+\n)/gm, maxLen: chunkSize },
+ { reg: /^(##\s[^\n]+\n)/gm, maxLen: chunkSize },
+ { reg: /^(###\s[^\n]+\n)/gm, maxLen: chunkSize },
+ { reg: /^(####\s[^\n]+\n)/gm, maxLen: chunkSize },
+ { reg: /^(#####\s[^\n]+\n)/gm, maxLen: chunkSize },
- { reg: /([\n]([`~]))/g, maxLen: chunkLen * 4 }, // code block
- { reg: /([\n](?=\s*[0-9]+\.))/g, maxLen: chunkLen * 2 }, // 增大块,尽可能保证它是一个完整的段落。 (?![\*\-|>`0-9]): markdown special char
- { reg: /(\n{2,})/g, maxLen: chunkLen * 1.6 },
- { reg: /([\n])/g, maxLen: chunkLen * 1.2 },
+ { reg: /([\n]([`~]))/g, maxLen: chunkSize }, // code block
+ { reg: /([\n](?=\s*[0-9]+\.))/g, maxLen: chunkSize }, // 增大块,尽可能保证它是一个完整的段落。 (?![\*\-|>`0-9]): markdown special char
+ { reg: /(\n{2,})/g, maxLen: chunkSize },
+ { reg: /([\n])/g, maxLen: chunkSize },
// ------ There's no overlap on the top
- { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.2 },
- { reg: /([!]|!\s)/g, maxLen: chunkLen * 1.2 },
- { reg: /([?]|\?\s)/g, maxLen: chunkLen * 1.4 },
- { reg: /([;]|;\s)/g, maxLen: chunkLen * 1.6 },
- { reg: /([,]|,\s)/g, maxLen: chunkLen * 2 }
+ { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkSize },
+ { reg: /([!]|!\s)/g, maxLen: chunkSize },
+ { reg: /([?]|\?\s)/g, maxLen: chunkSize },
+ { reg: /([;]|;\s)/g, maxLen: chunkSize },
+ { reg: /([,]|,\s)/g, maxLen: chunkSize }
];
const customRegLen = customReg.length;
@@ -203,7 +211,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
/* Gets the overlap at the end of a text as the beginning of the next block */
const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
const forbidOverlap = checkForbidOverlap(step);
- const maxOverlapLen = chunkLen * 0.4;
+ const maxOverlapLen = chunkSize * 0.4;
// step >= stepReges.length: Do not overlap incomplete sentences
if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
@@ -246,13 +254,13 @@ const commonSplit = (props: SplitProps): SplitResponse => {
// oversize
if (step >= stepReges.length) {
- if (text.length < chunkLen * 3) {
+ if (text.length < chunkSize * 3) {
return [text];
}
- // use slice-chunkLen to split text
+ // use slice-chunkSize to split text
const chunks: string[] = [];
- for (let i = 0; i < text.length; i += chunkLen - overlapLen) {
- chunks.push(text.slice(i, i + chunkLen));
+ for (let i = 0; i < text.length; i += chunkSize - overlapLen) {
+ chunks.push(text.slice(i, i + chunkSize));
}
return chunks;
}
@@ -260,8 +268,8 @@ const commonSplit = (props: SplitProps): SplitResponse => {
// split text by special char
const splitTexts = getSplitTexts({ text, step });
- const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkLen;
- const minChunkLen = chunkLen * 0.7;
+ const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkSize;
+ const minChunkLen = chunkSize * 0.7;
const chunks: string[] = [];
for (let i = 0; i < splitTexts.length; i++) {
@@ -297,7 +305,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
continue;
}
- // newText is too large(now, The lastText must be smaller than chunkLen)
+ // newText is too large(now, The lastText must be smaller than chunkSize)
if (newTextLen > maxLen) {
// lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
if (lastTextLen > minChunkLen) {
@@ -352,7 +360,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
/* If the last chunk is independent, it needs to be push chunks. */
if (lastText && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastText)) {
- if (lastText.length < chunkLen * 0.4) {
+ if (lastText.length < chunkSize * 0.4) {
chunks[chunks.length - 1] = chunks[chunks.length - 1] + lastText;
} else {
chunks.push(lastText);
@@ -386,9 +394,9 @@ const commonSplit = (props: SplitProps): SplitResponse => {
/**
* text split into chunks
- * chunkLen - one chunk len. max: 3500
+ * chunkSize - one chunk len. max: 3500
* overlapLen - The size of the before and after Text
- * chunkLen > overlapLen
+ * chunkSize > overlapLen
* markdown
*/
export const splitText2Chunks = (props: SplitProps): SplitResponse => {
diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts
index 99b4aaa3a..40e696b2d 100644
--- a/packages/global/core/dataset/api.d.ts
+++ b/packages/global/core/dataset/api.d.ts
@@ -1,5 +1,10 @@
import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
-import { DatasetCollectionTypeEnum, DatasetCollectionDataProcessModeEnum } from './constants';
+import {
+ DatasetCollectionTypeEnum,
+ DatasetCollectionDataProcessModeEnum,
+ ChunkSettingModeEnum,
+ DataChunkSplitModeEnum
+} from './constants';
import type { LLMModelItemType } from '../ai/model.d';
import { ParentIdType } from 'common/parentFolder/type';
@@ -33,7 +38,13 @@ export type DatasetCollectionChunkMetadataType = {
trainingType?: DatasetCollectionDataProcessModeEnum;
imageIndex?: boolean;
autoIndexes?: boolean;
+
+ chunkSettingMode?: ChunkSettingModeEnum;
+ chunkSplitMode?: DataChunkSplitModeEnum;
+
chunkSize?: number;
+ indexSize?: number;
+
chunkSplitter?: string;
qaPrompt?: string;
metadata?: Record;
diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts
index 81f52a4fe..627129835 100644
--- a/packages/global/core/dataset/constants.ts
+++ b/packages/global/core/dataset/constants.ts
@@ -129,6 +129,16 @@ export const DatasetCollectionDataProcessModeMap = {
}
};
+export enum ChunkSettingModeEnum {
+ auto = 'auto',
+ custom = 'custom'
+}
+
+export enum DataChunkSplitModeEnum {
+ size = 'size',
+ char = 'char'
+}
+
/* ------------ data -------------- */
/* ------------ training -------------- */
diff --git a/packages/global/core/dataset/controller.d.ts b/packages/global/core/dataset/controller.d.ts
index 2382a94e6..7a90ae5eb 100644
--- a/packages/global/core/dataset/controller.d.ts
+++ b/packages/global/core/dataset/controller.d.ts
@@ -13,6 +13,7 @@ export type CreateDatasetDataProps = {
export type UpdateDatasetDataProps = {
dataId: string;
+
q?: string;
a?: string;
indexes?: (Omit & {
diff --git a/packages/global/core/dataset/training/type.d.ts b/packages/global/core/dataset/training/type.d.ts
index 1bc15ea22..3404f1dc4 100644
--- a/packages/global/core/dataset/training/type.d.ts
+++ b/packages/global/core/dataset/training/type.d.ts
@@ -15,6 +15,8 @@ export type PushDataToTrainingQueueProps = {
vectorModel: string;
vlmModel?: string;
+ indexSize?: number;
+
billId?: string;
session?: ClientSession;
};
diff --git a/packages/global/core/dataset/training/utils.ts b/packages/global/core/dataset/training/utils.ts
new file mode 100644
index 000000000..895837abb
--- /dev/null
+++ b/packages/global/core/dataset/training/utils.ts
@@ -0,0 +1,136 @@
+import { EmbeddingModelItemType, LLMModelItemType } from '../../../core/ai/model.d';
+import {
+ ChunkSettingModeEnum,
+ DataChunkSplitModeEnum,
+ DatasetCollectionDataProcessModeEnum
+} from '../constants';
+
+export const minChunkSize = 64; // min index and chunk size
+
+// Chunk size
+export const chunkAutoChunkSize = 1500;
+export const getMaxChunkSize = (model: LLMModelItemType) => {
+ return Math.max(model.maxContext - model.maxResponse, 2000);
+};
+
+// QA
+export const defaultMaxChunkSize = 8000;
+export const getLLMDefaultChunkSize = (model?: LLMModelItemType) => {
+ if (!model) return defaultMaxChunkSize;
+ return Math.max(Math.min(model.maxContext - model.maxResponse, defaultMaxChunkSize), 2000);
+};
+
+export const getLLMMaxChunkSize = (model?: LLMModelItemType) => {
+ if (!model) return 8000;
+ return Math.max(model.maxContext - model.maxResponse, 2000);
+};
+
+// Index size
+export const getMaxIndexSize = (model?: EmbeddingModelItemType) => {
+ return model?.maxToken || 512;
+};
+export const getAutoIndexSize = (model?: EmbeddingModelItemType) => {
+ return model?.defaultToken || 512;
+};
+
+const indexSizeSelectList = [
+ {
+ label: '64',
+ value: 64
+ },
+ {
+ label: '128',
+ value: 128
+ },
+ {
+ label: '256',
+ value: 256
+ },
+ {
+ label: '512',
+ value: 512
+ },
+ {
+ label: '768',
+ value: 768
+ },
+ {
+ label: '1024',
+ value: 1024
+ },
+ {
+ label: '1536',
+ value: 1536
+ },
+ {
+ label: '2048',
+ value: 2048
+ },
+ {
+ label: '3072',
+ value: 3072
+ },
+ {
+ label: '4096',
+ value: 4096
+ },
+ {
+ label: '5120',
+ value: 5120
+ },
+ {
+ label: '6144',
+ value: 6144
+ },
+ {
+ label: '7168',
+ value: 7168
+ },
+ {
+ label: '8192',
+ value: 8192
+ }
+];
+export const getIndexSizeSelectList = (max = 512) => {
+ return indexSizeSelectList.filter((item) => item.value <= max);
+};
+
+// Compute
+export const computeChunkSize = (params: {
+ trainingType: DatasetCollectionDataProcessModeEnum;
+ chunkSettingMode?: ChunkSettingModeEnum;
+ chunkSplitMode?: DataChunkSplitModeEnum;
+ llmModel?: LLMModelItemType;
+ chunkSize?: number;
+}) => {
+ if (params.trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+ if (params.chunkSettingMode === ChunkSettingModeEnum.auto) {
+ return getLLMDefaultChunkSize(params.llmModel);
+ }
+ } else {
+ // chunk
+ if (params.chunkSettingMode === ChunkSettingModeEnum.auto) {
+ return chunkAutoChunkSize;
+ }
+ }
+
+ if (params.chunkSplitMode === DataChunkSplitModeEnum.char) {
+ return getLLMMaxChunkSize(params.llmModel);
+ }
+
+ return Math.min(params.chunkSize || chunkAutoChunkSize, getLLMMaxChunkSize(params.llmModel));
+};
+
+export const computeChunkSplitter = (params: {
+ chunkSettingMode?: ChunkSettingModeEnum;
+ chunkSplitMode?: DataChunkSplitModeEnum;
+ chunkSplitter?: string;
+}) => {
+ if (params.chunkSettingMode === ChunkSettingModeEnum.auto) {
+ return undefined;
+ }
+ if (params.chunkSplitMode === DataChunkSplitModeEnum.size) {
+ return undefined;
+ }
+ return params.chunkSplitter;
+};
diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts
index c9b0aa899..a92785b94 100644
--- a/packages/global/core/dataset/type.d.ts
+++ b/packages/global/core/dataset/type.d.ts
@@ -2,6 +2,7 @@ import type { LLMModelItemType, EmbeddingModelItemType } from '../../core/ai/mod
import { PermissionTypeEnum } from '../../support/permission/constant';
import { PushDatasetDataChunkProps } from './api';
import {
+ DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum,
DatasetStatusEnum,
@@ -14,6 +15,7 @@ import { Permission } from '../../support/permission/controller';
import { APIFileServer, FeishuServer, YuqueServer } from './apiDataset';
import { SourceMemberType } from 'support/user/type';
import { DatasetDataIndexTypeEnum } from './data/constants';
+import { ChunkSettingModeEnum } from './constants';
export type DatasetSchemaType = {
_id: string;
@@ -88,7 +90,12 @@ export type DatasetCollectionSchemaType = {
autoIndexes?: boolean;
imageIndex?: boolean;
trainingType: DatasetCollectionDataProcessModeEnum;
- chunkSize: number;
+
+ chunkSettingMode?: ChunkSettingModeEnum;
+ chunkSplitMode?: DataChunkSplitModeEnum;
+
+ chunkSize?: number;
+ indexSize?: number;
chunkSplitter?: string;
qaPrompt?: string;
};
diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts
index 64c330c84..803dce59b 100644
--- a/packages/global/core/dataset/utils.ts
+++ b/packages/global/core/dataset/utils.ts
@@ -1,7 +1,6 @@
import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constants';
import { getFileIcon } from '../../common/file/icon';
import { strIsLink } from '../../common/string/tools';
-import { DatasetDataIndexTypeEnum } from './data/constants';
export function getCollectionIcon(
type: DatasetCollectionTypeEnum = DatasetCollectionTypeEnum.file,
@@ -38,26 +37,6 @@ export function getSourceNameIcon({
return 'file/fill/file';
}
-/* get dataset data default index */
-export function getDefaultIndex(props?: { q?: string; a?: string }) {
- const { q = '', a } = props || {};
-
- return [
- {
- text: q,
- type: DatasetDataIndexTypeEnum.default
- },
- ...(a
- ? [
- {
- text: a,
- type: DatasetDataIndexTypeEnum.default
- }
- ]
- : [])
- ];
-}
-
export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
if (mode === TrainingModeEnum.qa) return data.length * 20;
if (mode === TrainingModeEnum.auto) return data.length * 5;
diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts
index 0dfcc6152..44e5d07da 100644
--- a/packages/service/core/dataset/collection/controller.ts
+++ b/packages/service/core/dataset/collection/controller.ts
@@ -27,6 +27,11 @@ import { addDays } from 'date-fns';
import { MongoDatasetDataText } from '../data/dataTextSchema';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getTrainingModeByCollection } from './utils';
+import {
+ computeChunkSize,
+ computeChunkSplitter,
+ getLLMMaxChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
export const createCollectionAndInsertData = async ({
dataset,
@@ -54,18 +59,22 @@ export const createCollectionAndInsertData = async ({
const teamId = createCollectionParams.teamId;
const tmbId = createCollectionParams.tmbId;
- // Chunk split params
+
+ // Set default params
const trainingType =
createCollectionParams.trainingType || DatasetCollectionDataProcessModeEnum.chunk;
- const chunkSize = createCollectionParams.chunkSize || 512;
- const chunkSplitter = createCollectionParams.chunkSplitter;
- const qaPrompt = createCollectionParams.qaPrompt;
- const usageName = createCollectionParams.name;
+ const chunkSize = computeChunkSize({
+ ...createCollectionParams,
+ trainingType,
+ llmModel: getLLMModel(dataset.agentModel)
+ });
+ const chunkSplitter = computeChunkSplitter(createCollectionParams);
// 1. split chunks
const chunks = rawText2Chunks({
rawText,
- chunkLen: chunkSize,
+ chunkSize,
+ maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
overlapRatio: trainingType === DatasetCollectionDataProcessModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : [],
isQAImport
@@ -76,7 +85,7 @@ export const createCollectionAndInsertData = async ({
teamId,
insertLen: predictDataLimitLength(
getTrainingModeByCollection({
- trainingType,
+ trainingType: trainingType,
autoIndexes: createCollectionParams.autoIndexes,
imageIndex: createCollectionParams.imageIndex
}),
@@ -88,6 +97,9 @@ export const createCollectionAndInsertData = async ({
// 3. create collection
const { _id: collectionId } = await createOneCollection({
...createCollectionParams,
+ trainingType,
+ chunkSize,
+ chunkSplitter,
hashRawText: hashStr(rawText),
rawTextLength: rawText.length,
@@ -111,7 +123,7 @@ export const createCollectionAndInsertData = async ({
const { billId: newBillId } = await createTrainingUsage({
teamId,
tmbId,
- appName: usageName,
+ appName: createCollectionParams.name,
billSource: UsageSourceEnum.training,
vectorModel: getEmbeddingModel(dataset.vectorModel)?.name,
agentModel: getLLMModel(dataset.agentModel)?.name,
@@ -130,12 +142,13 @@ export const createCollectionAndInsertData = async ({
agentModel: dataset.agentModel,
vectorModel: dataset.vectorModel,
vlmModel: dataset.vlmModel,
+ indexSize: createCollectionParams.indexSize,
mode: getTrainingModeByCollection({
- trainingType,
+ trainingType: trainingType,
autoIndexes: createCollectionParams.autoIndexes,
imageIndex: createCollectionParams.imageIndex
}),
- prompt: qaPrompt,
+ prompt: createCollectionParams.qaPrompt,
billId: traingBillId,
data: chunks.map((item, index) => ({
...item,
@@ -207,11 +220,14 @@ export async function createOneCollection({
// Parse settings
customPdfParse,
imageIndex,
+ autoIndexes,
// Chunk settings
- trainingType = DatasetCollectionDataProcessModeEnum.chunk,
- autoIndexes,
- chunkSize = 512,
+ trainingType,
+ chunkSettingMode,
+ chunkSplitMode,
+ chunkSize,
+ indexSize,
chunkSplitter,
qaPrompt,
@@ -249,11 +265,14 @@ export async function createOneCollection({
// Parse settings
customPdfParse,
imageIndex,
+ autoIndexes,
// Chunk settings
trainingType,
- autoIndexes,
+ chunkSettingMode,
+ chunkSplitMode,
chunkSize,
+ indexSize,
chunkSplitter,
qaPrompt
}
diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts
index 7e1686f95..9522c69f2 100644
--- a/packages/service/core/dataset/collection/schema.ts
+++ b/packages/service/core/dataset/collection/schema.ts
@@ -3,7 +3,9 @@ const { Schema, model, models } = connectionMongo;
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
import {
DatasetCollectionTypeMap,
- DatasetCollectionDataProcessModeEnum
+ DatasetCollectionDataProcessModeEnum,
+ ChunkSettingModeEnum,
+ DataChunkSplitModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionName } from '../schema';
import {
@@ -94,11 +96,18 @@ const DatasetCollectionSchema = new Schema({
type: String,
enum: Object.values(DatasetCollectionDataProcessModeEnum)
},
- chunkSize: {
- type: Number,
- required: true
+ chunkSettingMode: {
+ type: String,
+ enum: Object.values(ChunkSettingModeEnum)
},
+ chunkSplitMode: {
+ type: String,
+ enum: Object.values(DataChunkSplitModeEnum)
+ },
+ chunkSize: Number,
chunkSplitter: String,
+
+ indexSize: Number,
qaPrompt: String
});
diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts
index 730e0dc6c..448396cc3 100644
--- a/packages/service/core/dataset/read.ts
+++ b/packages/service/core/dataset/read.ts
@@ -185,7 +185,7 @@ export const readApiServerFileContent = async ({
export const rawText2Chunks = ({
rawText,
isQAImport,
- chunkLen = 512,
+ chunkSize = 512,
...splitProps
}: {
rawText: string;
@@ -198,7 +198,7 @@ export const rawText2Chunks = ({
const { chunks } = splitText2Chunks({
text: rawText,
- chunkLen,
+ chunkSize,
...splitProps
});
diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts
index d740eec55..fb5ceacd8 100644
--- a/packages/service/core/dataset/training/controller.ts
+++ b/packages/service/core/dataset/training/controller.ts
@@ -12,6 +12,10 @@ import { getCollectionWithDataset } from '../controller';
import { mongoSessionRun } from '../../../common/mongo/sessionRun';
import { PushDataToTrainingQueueProps } from '@fastgpt/global/core/dataset/training/type';
import { i18nT } from '../../../../web/i18n/utils';
+import {
+ getLLMDefaultChunkSize,
+ getLLMMaxChunkSize
+} from '../../../../global/core/dataset/training/utils';
export const lockTrainingDataByTeamId = async (teamId: string): Promise => {
try {
@@ -55,6 +59,7 @@ export async function pushDataListToTrainingQueue({
prompt,
billId,
mode = TrainingModeEnum.chunk,
+ indexSize,
session
}: PushDataToTrainingQueueProps): Promise {
const getImageChunkMode = (data: PushDatasetDataChunkProps, mode: TrainingModeEnum) => {
@@ -68,38 +73,41 @@ export async function pushDataListToTrainingQueue({
}
return mode;
};
+
+ const vectorModelData = getEmbeddingModel(vectorModel);
+ if (!vectorModelData) {
+ return Promise.reject(i18nT('common:error_embedding_not_config'));
+ }
+ const agentModelData = getLLMModel(agentModel);
+ if (!agentModelData) {
+ return Promise.reject(i18nT('common:error_llm_not_config'));
+ }
+ if (mode === TrainingModeEnum.chunk || mode === TrainingModeEnum.auto) {
+ prompt = undefined;
+ }
+
const { model, maxToken, weight } = await (async () => {
if (mode === TrainingModeEnum.chunk) {
- const vectorModelData = getEmbeddingModel(vectorModel);
- if (!vectorModelData) {
- return Promise.reject(i18nT('common:error_embedding_not_config'));
- }
return {
- maxToken: vectorModelData.maxToken * 1.5,
+ maxToken: getLLMMaxChunkSize(agentModelData),
model: vectorModelData.model,
weight: vectorModelData.weight
};
}
-
if (mode === TrainingModeEnum.qa || mode === TrainingModeEnum.auto) {
- const agentModelData = getLLMModel(agentModel);
- if (!agentModelData) {
- return Promise.reject(i18nT('common:error_llm_not_config'));
- }
return {
- maxToken: agentModelData.maxContext * 0.8,
+ maxToken: getLLMMaxChunkSize(agentModelData),
model: agentModelData.model,
weight: 0
};
}
-
if (mode === TrainingModeEnum.image) {
const vllmModelData = getVlmModel(vlmModel);
if (!vllmModelData) {
return Promise.reject(i18nT('common:error_vlm_not_config'));
}
return {
- maxToken: vllmModelData.maxContext * 0.8,
+ maxToken: getLLMMaxChunkSize(vllmModelData),
model: vllmModelData.model,
weight: 0
};
@@ -107,10 +115,6 @@ export async function pushDataListToTrainingQueue({
return Promise.reject(`Training mode "${mode}" is inValid`);
})();
- // Filter redundant params
- if (mode === TrainingModeEnum.chunk || mode === TrainingModeEnum.auto) {
- prompt = undefined;
- }
// filter repeat or equal content
const set = new Set();
@@ -143,13 +147,13 @@ export async function pushDataListToTrainingQueue({
const text = item.q + item.a;
+ // Oversize llm tokens
if (text.length > maxToken) {
filterResult.overToken.push(item);
return;
}
if (set.has(text)) {
- console.log('repeat', item);
filterResult.repeat.push(item);
} else {
filterResult.success.push(item);
@@ -182,6 +186,7 @@ export async function pushDataListToTrainingQueue({
q: item.q,
a: item.a,
chunkIndex: item.chunkIndex ?? 0,
+ indexSize,
weight: weight ?? 0,
indexes: item.indexes,
retryCount: 5
diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts
index 34044674d..d11d2e109 100644
--- a/packages/service/core/dataset/training/schema.ts
+++ b/packages/service/core/dataset/training/schema.ts
@@ -76,6 +76,7 @@ const TrainingDataSchema = new Schema({
type: Number,
default: 0
},
+ indexSize: Number,
weight: {
type: Number,
default: 0
diff --git a/packages/web/components/common/MyModal/EditFolderModal.tsx b/packages/web/components/common/MyModal/EditFolderModal.tsx
index bc8dfdda9..4647066c6 100644
--- a/packages/web/components/common/MyModal/EditFolderModal.tsx
+++ b/packages/web/components/common/MyModal/EditFolderModal.tsx
@@ -72,7 +72,7 @@ const EditFolderModal = ({
{...register('name', { required: true })}
bg={'myGray.50'}
autoFocus
- maxLength={20}
+ maxLength={100}
/>
diff --git a/packages/web/components/common/Radio/RadioGroup.tsx b/packages/web/components/common/Radio/RadioGroup.tsx
new file mode 100644
index 000000000..1443a09dd
--- /dev/null
+++ b/packages/web/components/common/Radio/RadioGroup.tsx
@@ -0,0 +1,67 @@
+import React from 'react';
+import { Box, Flex, Grid, type GridProps, HStack } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import QuestionTip from '../MyTooltip/QuestionTip';
+
+type Props = Omit & {
+ list: {
+ title: string;
+ value: T;
+ tooltip?: string;
+ }[];
+ value: T;
+ defaultBg?: string;
+ activeBg?: string;
+ onChange: (e: T) => void;
+};
+
+const RadioGroup = ({ list, value, onChange, ...props }: Props) => {
+ const { t } = useTranslation();
+
+ return (
+
+ {list.map((item) => (
+ onChange(item.value)}
+ >
+
+
+
+
+
+
+ {typeof item.title === 'string' ? t(item.title as any) : item.title}
+ {!!item.tooltip && }
+
+
+ ))}
+
+ );
+};
+
+export default RadioGroup;
diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json
index 36e22c152..5581d22e2 100644
--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
@@ -569,7 +569,6 @@
"core.dataset.import.Custom process": "Custom Rules",
"core.dataset.import.Custom process desc": "Customize segmentation and preprocessing rules",
"core.dataset.import.Custom prompt": "Custom Prompt",
- "core.dataset.import.Custom split char": "Custom Separator",
"core.dataset.import.Custom text": "Custom Text",
"core.dataset.import.Custom text desc": "Manually enter a piece of text as a dataset",
"core.dataset.import.Data process params": "Data Processing Parameters",
diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json
index d894d21c8..67a961810 100644
--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -27,7 +27,6 @@
"custom_data_process_params": "Custom",
"custom_data_process_params_desc": "Customize data processing rules",
"custom_split_sign_tip": "Allows you to chunk according to custom delimiters. \nUsually used for processed data, using specific separators for precise chunking. \nYou can use the | symbol to represent multiple splitters, such as: \".|.\" to represent a period in Chinese and English.\n\nTry to avoid using special symbols related to regular, such as: * () [] {}, etc.",
- "data.ideal_chunk_length": "ideal block length",
"data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes",
"data_index_num": "Index {{index}}",
"data_process_params": "Params",
@@ -53,8 +52,6 @@
"file_model_function_tip": "Enhances indexing and QA generation",
"filename": "Filename",
"folder_dataset": "Folder",
- "ideal_chunk_length": "ideal block length",
- "ideal_chunk_length_tips": "Segment according to the end symbol and combine multiple segments into one block. This value determines the estimated size of the block, if there is any fluctuation.",
"image_auto_parse": "Automatic image indexing",
"image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
"image_training_queue": "Queue of image processing",
@@ -68,6 +65,8 @@
"import_param_setting": "Parameter settings",
"import_select_file": "Select a file",
"import_select_link": "Enter link",
+ "index_size": "Index size",
+ "index_size_tips": "When vectorized, the system will automatically further segment the blocks according to this size.",
"is_open_schedule": "Enable scheduled synchronization",
"keep_image": "Keep the picture",
"move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.",
@@ -89,6 +88,8 @@
"retain_collection": "Adjust Training Parameters",
"retrain_task_submitted": "The retraining task has been submitted",
"same_api_collection": "The same API set exists",
+ "split_chunk_char": "Block by specified splitter",
+ "split_chunk_size": "Block by length",
"split_sign_break": "1 newline character",
"split_sign_break2": "2 newline characters",
"split_sign_custom": "Customize",
diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json
index 0e6767d34..c7068843b 100644
--- a/packages/web/i18n/zh-CN/common.json
+++ b/packages/web/i18n/zh-CN/common.json
@@ -573,7 +573,6 @@
"core.dataset.import.Custom process": "自定义规则",
"core.dataset.import.Custom process desc": "自定义设置数据处理规则",
"core.dataset.import.Custom prompt": "自定义提示词",
- "core.dataset.import.Custom split char": "自定义分隔符",
"core.dataset.import.Custom text": "自定义文本",
"core.dataset.import.Custom text desc": "手动输入一段文本作为数据集",
"core.dataset.import.Data process params": "数据处理参数",
diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json
index bc4915917..f8af7612b 100644
--- a/packages/web/i18n/zh-CN/dataset.json
+++ b/packages/web/i18n/zh-CN/dataset.json
@@ -27,7 +27,6 @@
"custom_data_process_params": "自定义",
"custom_data_process_params_desc": "自定义设置数据处理规则",
"custom_split_sign_tip": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据,使用特定的分隔符来精确分块。可以使用 | 符号表示多个分割符,例如:“。|.” 表示中英文句号。\n尽量避免使用正则相关特殊符号,例如: * () [] {} 等。",
- "data.ideal_chunk_length": "理想分块长度",
"data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
"data_index_num": "索引 {{index}}",
"data_process_params": "处理参数",
@@ -53,8 +52,6 @@
"file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名",
"folder_dataset": "文件夹",
- "ideal_chunk_length": "理想分块长度",
- "ideal_chunk_length_tips": "按结束符号进行分段,并将多个分段组成一个分块,该值决定了分块的预估大小,如果会有上下浮动。",
"image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
"image_training_queue": "图片处理排队",
@@ -68,6 +65,8 @@
"import_param_setting": "参数设置",
"import_select_file": "选择文件",
"import_select_link": "输入链接",
+ "index_size": "索引大小",
+ "index_size_tips": "向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。",
"is_open_schedule": "启用定时同步",
"keep_image": "保留图片",
"move.hint": "移动后,所选知识库/文件夹将继承新文件夹的权限设置,原先的权限设置失效。",
@@ -89,6 +88,8 @@
"retain_collection": "调整训练参数",
"retrain_task_submitted": "重新训练任务已提交",
"same_api_collection": "存在相同的 API 集合",
+ "split_chunk_char": "按指定分割符分块",
+ "split_chunk_size": "按长度分块",
"split_sign_break": "1 个换行符",
"split_sign_break2": "2 个换行符",
"split_sign_custom": "自定义",
diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json
index 14e12b5e6..2d3336cc7 100644
--- a/packages/web/i18n/zh-Hant/common.json
+++ b/packages/web/i18n/zh-Hant/common.json
@@ -568,7 +568,6 @@
"core.dataset.import.Custom process": "自訂規則",
"core.dataset.import.Custom process desc": "自訂設定資料處理規則",
"core.dataset.import.Custom prompt": "自訂提示詞",
- "core.dataset.import.Custom split char": "自訂分隔符",
"core.dataset.import.Custom text": "自訂文字",
"core.dataset.import.Custom text desc": "手動輸入一段文字作為資料集",
"core.dataset.import.Data process params": "資料處理參數",
diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json
index d1c3404f7..1f687b594 100644
--- a/packages/web/i18n/zh-Hant/dataset.json
+++ b/packages/web/i18n/zh-Hant/dataset.json
@@ -27,7 +27,6 @@
"custom_data_process_params": "自訂",
"custom_data_process_params_desc": "自訂資料處理規則",
"custom_split_sign_tip": "允許你根據自定義的分隔符進行分塊。\n通常用於已處理好的數據,使用特定的分隔符來精確分塊。\n可以使用 | 符號表示多個分割符,例如:“。|.” 表示中英文句號。\n\n盡量避免使用正則相關特殊符號,例如: * () [] {} 等。",
- "data.ideal_chunk_length": "理想分塊長度",
"data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引",
"data_index_num": "索引 {{index}}",
"data_process_params": "處理參數",
@@ -53,8 +52,6 @@
"file_model_function_tip": "用於增強索引和問答生成",
"filename": "檔案名稱",
"folder_dataset": "資料夾",
- "ideal_chunk_length": "理想分塊長度",
- "ideal_chunk_length_tips": "依結束符號進行分段,並將多個分段組成一個分塊,此值決定了分塊的預估大小,可能會有上下浮動。",
"image_auto_parse": "圖片自動索引",
"image_auto_parse_tips": "調用 VLM 自動標註文檔裡的圖片,並生成額外的檢索索引",
"image_training_queue": "圖片處理排隊",
@@ -68,6 +65,8 @@
"import_param_setting": "參數設置",
"import_select_file": "選擇文件",
"import_select_link": "輸入鏈接",
+ "index_size": "索引大小",
+ "index_size_tips": "向量化時內容的長度,系統會自動按該大小對分塊進行進一步的分割。",
"is_open_schedule": "啟用定時同步",
"keep_image": "保留圖片",
"move.hint": "移動後,所選資料集/資料夾將繼承新資料夾的權限設定,原先的權限設定將失效。",
@@ -89,6 +88,8 @@
"retain_collection": "調整訓練參數",
"retrain_task_submitted": "重新訓練任務已提交",
"same_api_collection": "存在相同的 API 集合",
+ "split_chunk_char": "按指定分割符分塊",
+ "split_chunk_size": "按長度分塊",
"split_sign_break": "1 個換行符",
"split_sign_break2": "2 個換行符",
"split_sign_custom": "自定義",
diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx
index 5fd5e0f4f..6236d8491 100644
--- a/projects/app/src/components/common/Modal/EditResourceModal.tsx
+++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx
@@ -71,7 +71,7 @@ const EditResourceModal = ({
{...register('name', { required: true })}
bg={'myGray.50'}
autoFocus
- maxLength={20}
+ maxLength={100}
/>
diff --git a/projects/app/src/components/support/apikey/Table.tsx b/projects/app/src/components/support/apikey/Table.tsx
index 30574b55b..96561c2b0 100644
--- a/projects/app/src/components/support/apikey/Table.tsx
+++ b/projects/app/src/components/support/apikey/Table.tsx
@@ -338,7 +338,7 @@ function EditKeyModal({
{t('common:Name')}
{t('common:Name')}
diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/NodeCard.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/NodeCard.tsx
index b7feff68f..f461ef888 100644
--- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/NodeCard.tsx
+++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/NodeCard.tsx
@@ -418,7 +418,7 @@ const NodeCard = (props: Props) => {
{RenderToolHandle}
-
+
);
};
diff --git a/projects/app/src/pageComponents/app/list/TemplateMarketModal.tsx b/projects/app/src/pageComponents/app/list/TemplateMarketModal.tsx
index fbcd1e2b2..9b2553ad3 100644
--- a/projects/app/src/pageComponents/app/list/TemplateMarketModal.tsx
+++ b/projects/app/src/pageComponents/app/list/TemplateMarketModal.tsx
@@ -319,7 +319,7 @@ const TemplateMarketModal = ({
onChange={(e) => setCurrentSearch(e.target.value)}
h={8}
bg={'myGray.50'}
- maxLength={20}
+ maxLength={100}
borderRadius={'sm'}
/>
diff --git a/projects/app/src/pageComponents/dataset/EditFolderModal.tsx b/projects/app/src/pageComponents/dataset/EditFolderModal.tsx
index 961e01f41..a17a51227 100644
--- a/projects/app/src/pageComponents/dataset/EditFolderModal.tsx
+++ b/projects/app/src/pageComponents/dataset/EditFolderModal.tsx
@@ -49,7 +49,7 @@ const EditFolderModal = ({
defaultValue={name}
placeholder={t('common:dataset.Folder Name') || ''}
autoFocus
- maxLength={20}
+ maxLength={100}
/>
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
index d437f39c1..29860701b 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
@@ -10,11 +10,21 @@ import { useMyStep } from '@fastgpt/web/hooks/useStep';
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { TabEnum } from '../NavBar';
-import { ChunkSettingModeEnum } from '@/web/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { UseFormReturn, useForm } from 'react-hook-form';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { DataChunkSplitModeEnum } from '@fastgpt/global/core/dataset/constants';
+import {
+ getMaxChunkSize,
+ getLLMDefaultChunkSize,
+ getLLMMaxChunkSize,
+ chunkAutoChunkSize,
+ minChunkSize,
+ getAutoIndexSize,
+ getMaxIndexSize
+} from '@fastgpt/global/core/dataset/training/utils';
type TrainingFiledType = {
chunkOverlapRatio: number;
@@ -22,6 +32,9 @@ type TrainingFiledType = {
minChunkSize: number;
autoChunkSize: number;
chunkSize: number;
+ maxIndexSize?: number;
+ indexSize?: number;
+ autoIndexSize?: number;
charsPointsPrice: number;
priceTip: string;
uploadRate: number;
@@ -47,9 +60,13 @@ export type ImportFormType = {
autoIndexes: boolean;
chunkSettingMode: ChunkSettingModeEnum;
+
+ chunkSplitMode: DataChunkSplitModeEnum;
embeddingChunkSize: number;
qaChunkSize: number;
- customSplitChar: string;
+ chunkSplitter: string;
+ indexSize: number;
+
qaPrompt: string;
webSelector: string;
};
@@ -199,9 +216,12 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
trainingType: DatasetCollectionDataProcessModeEnum.chunk,
chunkSettingMode: ChunkSettingModeEnum.auto,
- embeddingChunkSize: vectorModel?.defaultToken || 512,
- qaChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
- customSplitChar: '',
+
+ chunkSplitMode: DataChunkSplitModeEnum.size,
+ embeddingChunkSize: 2000,
+ indexSize: vectorModel?.defaultToken || 512,
+ qaChunkSize: getLLMDefaultChunkSize(agentModel),
+ chunkSplitter: '',
qaPrompt: Prompt_AgentQA.description,
webSelector: '',
customPdfParse: false
@@ -215,17 +235,18 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
const chunkSettingMode = processParamsForm.watch('chunkSettingMode');
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
const qaChunkSize = processParamsForm.watch('qaChunkSize');
- const customSplitChar = processParamsForm.watch('customSplitChar');
+ const chunkSplitter = processParamsForm.watch('chunkSplitter');
const autoIndexes = processParamsForm.watch('autoIndexes');
+ const indexSize = processParamsForm.watch('indexSize');
const TrainingModeMap = useMemo(() => {
if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
return {
chunkSizeField: 'qaChunkSize',
chunkOverlapRatio: 0,
- maxChunkSize: Math.min(agentModel.maxResponse * 4, agentModel.maxContext * 0.7),
- minChunkSize: 4000,
- autoChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
+ maxChunkSize: getLLMMaxChunkSize(agentModel),
+ minChunkSize: 1000,
+ autoChunkSize: getLLMDefaultChunkSize(agentModel),
chunkSize: qaChunkSize,
charsPointsPrice: agentModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
@@ -237,10 +258,13 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
return {
chunkSizeField: 'embeddingChunkSize',
chunkOverlapRatio: 0.2,
- maxChunkSize: 2048,
- minChunkSize: 100,
- autoChunkSize: vectorModel?.defaultToken ? vectorModel.defaultToken * 2 : 1024,
+ maxChunkSize: getMaxChunkSize(agentModel),
+ minChunkSize: minChunkSize,
+ autoChunkSize: chunkAutoChunkSize,
chunkSize: embeddingChunkSize,
+ maxIndexSize: getMaxIndexSize(vectorModel),
+ autoIndexSize: getAutoIndexSize(vectorModel),
+ indexSize,
charsPointsPrice: agentModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
price: agentModel.charsPointsPrice
@@ -251,10 +275,13 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
return {
chunkSizeField: 'embeddingChunkSize',
chunkOverlapRatio: 0.2,
- maxChunkSize: vectorModel?.maxToken || 512,
- minChunkSize: 100,
- autoChunkSize: vectorModel?.defaultToken || 512,
+ maxChunkSize: getMaxChunkSize(agentModel),
+ minChunkSize: minChunkSize,
+ autoChunkSize: chunkAutoChunkSize,
chunkSize: embeddingChunkSize,
+ maxIndexSize: getMaxIndexSize(vectorModel),
+ autoIndexSize: getAutoIndexSize(vectorModel),
+ indexSize,
charsPointsPrice: vectorModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Embedding Estimated Price Tips', {
price: vectorModel.charsPointsPrice
@@ -265,30 +292,36 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
}, [
trainingType,
autoIndexes,
- agentModel.maxResponse,
- agentModel.maxContext,
- agentModel.charsPointsPrice,
+ agentModel,
qaChunkSize,
t,
- vectorModel.defaultToken,
- vectorModel?.maxToken,
- vectorModel.charsPointsPrice,
- embeddingChunkSize
+ embeddingChunkSize,
+ vectorModel,
+ indexSize
]);
const chunkSettingModeMap = useMemo(() => {
if (chunkSettingMode === ChunkSettingModeEnum.auto) {
return {
chunkSize: TrainingModeMap.autoChunkSize,
- customSplitChar: ''
+ indexSize: TrainingModeMap.autoIndexSize,
+ chunkSplitter: ''
};
} else {
return {
chunkSize: TrainingModeMap.chunkSize,
- customSplitChar
+ indexSize: TrainingModeMap.indexSize,
+ chunkSplitter
};
}
- }, [chunkSettingMode, TrainingModeMap.autoChunkSize, TrainingModeMap.chunkSize, customSplitChar]);
+ }, [
+ chunkSettingMode,
+ TrainingModeMap.autoChunkSize,
+ TrainingModeMap.autoIndexSize,
+ TrainingModeMap.chunkSize,
+ TrainingModeMap.indexSize,
+ chunkSplitter
+ ]);
const contextValue = {
...TrainingModeMap,
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
index 5a1dd5065..d9366a209 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
@@ -20,10 +20,11 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import {
+ DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionDataProcessModeMap
} from '@fastgpt/global/core/dataset/constants';
-import { ChunkSettingModeEnum } from '@/web/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyModal from '@fastgpt/web/components/common/MyModal';
@@ -37,25 +38,39 @@ import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { shadowLight } from '@fastgpt/web/styles/theme';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MySelect from '@fastgpt/web/components/common/MySelect';
+import { getIndexSizeSelectList } from '@fastgpt/global/core/dataset/training/utils';
+import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
function DataProcess() {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
- const { goToNext, processParamsForm, chunkSizeField, minChunkSize, maxChunkSize } =
- useContextSelector(DatasetImportContext, (v) => v);
+ const {
+ goToNext,
+ processParamsForm,
+ chunkSizeField,
+ minChunkSize,
+ maxChunkSize,
+ maxIndexSize,
+ indexSize
+ } = useContextSelector(DatasetImportContext, (v) => v);
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { setValue, register, watch, getValues } = processParamsForm;
const trainingType = watch('trainingType');
- const chunkSettingMode = watch('chunkSettingMode');
+ const trainingModeList = useMemo(() => {
+ const list = Object.entries(DatasetCollectionDataProcessModeMap);
+ return list
+ .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
+ .map(([key, value]) => ({
+ title: t(value.label as any),
+ value: key as DatasetCollectionDataProcessModeEnum,
+ tooltip: t(value.tooltip as any)
+ }));
+ }, [t]);
- const qaPrompt = watch('qaPrompt');
- const {
- isOpen: isOpenCustomPrompt,
- onOpen: onOpenCustomPrompt,
- onClose: onCloseCustomPrompt
- } = useDisclosure();
+ const chunkSettingMode = watch('chunkSettingMode');
+ const chunkSplitMode = watch('chunkSplitMode');
const customSplitList = [
{ label: t('dataset:split_sign_null'), value: '' },
@@ -69,25 +84,25 @@ function DataProcess() {
{ label: t('dataset:split_sign_custom'), value: 'Other' }
];
- const [customListSelectValue, setCustomListSelectValue] = useState(getValues('customSplitChar'));
+ const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
useEffect(() => {
if (customListSelectValue === 'Other') {
- setValue('customSplitChar', '');
+ setValue('chunkSplitter', '');
} else {
- setValue('customSplitChar', customListSelectValue);
+ setValue('chunkSplitter', customListSelectValue);
}
}, [customListSelectValue, setValue]);
- const trainingModeList = useMemo(() => {
- const list = Object.entries(DatasetCollectionDataProcessModeMap);
- return list
- .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
- .map(([key, value]) => ({
- title: t(value.label as any),
- value: key as DatasetCollectionDataProcessModeEnum,
- tooltip: t(value.tooltip as any)
- }));
- }, [t]);
+ // Index size
+ const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
+
+ // QA
+ const qaPrompt = watch('qaPrompt');
+ const {
+ isOpen: isOpenCustomPrompt,
+ onOpen: onOpenCustomPrompt,
+ onClose: onCloseCustomPrompt
+ } = useDisclosure();
const Title = useCallback(({ title }: { title: string }) => {
return (
@@ -237,67 +252,97 @@ function DataProcess() {
children: chunkSettingMode === ChunkSettingModeEnum.custom && (
-
- {t('dataset:ideal_chunk_length')}
-
-
- span': {
- display: 'block'
+
+ list={[
+ {
+ title: t('dataset:split_chunk_size'),
+ value: DataChunkSplitModeEnum.size
+ },
+ {
+ title: t('dataset:split_chunk_char'),
+ value: DataChunkSplitModeEnum.char,
+ tooltip: t('dataset:custom_split_sign_tip')
}
+ ]}
+ value={chunkSplitMode}
+ onChange={(e) => {
+ setValue('chunkSplitMode', e);
}}
- >
-
+
+ {chunkSplitMode === DataChunkSplitModeEnum.size && (
+ span': {
+ display: 'block'
+ }
+ }}
>
-
-
-
+
+
+
+
+ )}
+
+ {chunkSplitMode === DataChunkSplitModeEnum.char && (
+
+
+
+ list={customSplitList}
+ size={'sm'}
+ bg={'myGray.50'}
+ value={customListSelectValue}
+ h={'32px'}
+ onChange={(val) => {
+ setCustomListSelectValue(val);
+ }}
+ />
+
+ {customListSelectValue === 'Other' && (
+
+ )}
+
+ )}
-
+ {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
- {t('common:core.dataset.import.Custom split char')}
-
-
-
-
-
-
- list={customSplitList}
- size={'sm'}
+
+ {t('dataset:index_size')}
+
+
+
+
bg={'myGray.50'}
- value={customListSelectValue}
- h={'32px'}
+ list={indexSizeSeletorList}
+ value={indexSize}
onChange={(val) => {
- setCustomListSelectValue(val);
+ setValue('indexSize', val);
}}
/>
- {customListSelectValue === 'Other' && (
-
- )}
-
-
+
+ )}
{showQAPromptInput && (
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
index 1b2ce5c23..37a09bc2c 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
@@ -16,6 +16,7 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
import MyBox from '@fastgpt/web/components/common/MyBox';
import Markdown from '@/components/Markdown';
import { useToast } from '@fastgpt/web/hooks/useToast';
+import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils';
const PreviewData = () => {
const { t } = useTranslation();
@@ -23,6 +24,7 @@ const PreviewData = () => {
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
+ const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const sources = useContextSelector(DatasetImportContext, (v) => v.sources);
const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource);
@@ -36,12 +38,13 @@ const PreviewData = () => {
async () => {
if (!previewFile) return;
if (importSource === ImportDataSourceEnum.fileCustom) {
- const customSplitChar = processParamsForm.getValues('customSplitChar');
+ const chunkSplitter = processParamsForm.getValues('chunkSplitter');
const { chunks } = splitText2Chunks({
text: previewFile.rawText || '',
- chunkLen: chunkSize,
+ chunkSize,
+ maxSize: getLLMMaxChunkSize(datasetDetail.agentModel),
overlapRatio: chunkOverlapRatio,
- customReg: customSplitChar ? [customSplitChar] : []
+ customReg: chunkSplitter ? [chunkSplitter] : []
});
return chunks.map((chunk) => ({
q: chunk,
@@ -61,9 +64,12 @@ const PreviewData = () => {
customPdfParse: processParamsForm.getValues('customPdfParse'),
+ trainingType: processParamsForm.getValues('trainingType'),
+ chunkSettingMode: processParamsForm.getValues('chunkSettingMode'),
+ chunkSplitMode: processParamsForm.getValues('chunkSplitMode'),
chunkSize,
+ chunkSplitter: processParamsForm.getValues('chunkSplitter'),
overlapRatio: chunkOverlapRatio,
- customSplitChar: processParamsForm.getValues('customSplitChar'),
selector: processParamsForm.getValues('webSelector'),
isQAImport: importSource === ImportDataSourceEnum.csvTable,
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx
index 489f9c0f2..45cab2504 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx
@@ -49,7 +49,7 @@ const Upload = () => {
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const retrainNewCollectionId = useRef('');
- const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } =
+ const { importSource, parentId, sources, setSources, processParamsForm, chunkSize, indexSize } =
useContextSelector(DatasetImportContext, (v) => v);
const { handleSubmit } = processParamsForm;
@@ -81,7 +81,7 @@ const Upload = () => {
}, [waitingFilesCount, totalFilesCount, allFinished, t]);
const { runAsync: startUpload, loading: isLoading } = useRequest2(
- async ({ trainingType, customSplitChar, qaPrompt, webSelector }: ImportFormType) => {
+ async ({ trainingType, chunkSplitter, qaPrompt, webSelector }: ImportFormType) => {
if (sources.length === 0) return;
const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting');
@@ -111,10 +111,16 @@ const Upload = () => {
trainingType,
imageIndex: processParamsForm.getValues('imageIndex'),
autoIndexes: processParamsForm.getValues('autoIndexes'),
+
+ chunkSettingMode: processParamsForm.getValues('chunkSettingMode'),
+ chunkSplitMode: processParamsForm.getValues('chunkSplitMode'),
+
chunkSize,
- chunkSplitter: customSplitChar,
+ indexSize,
+ chunkSplitter,
qaPrompt: trainingType === DatasetCollectionDataProcessModeEnum.qa ? qaPrompt : undefined
};
+
if (importSource === ImportDataSourceEnum.reTraining) {
const res = await postReTrainingDatasetFileCollection({
...commonParams,
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/components/PreviewChunks.tsx b/projects/app/src/pageComponents/dataset/detail/Import/components/PreviewChunks.tsx
deleted file mode 100644
index 1248764bb..000000000
--- a/projects/app/src/pageComponents/dataset/detail/Import/components/PreviewChunks.tsx
+++ /dev/null
@@ -1,102 +0,0 @@
-import React from 'react';
-import { Box } from '@chakra-ui/react';
-import { ImportSourceItemType } from '@/web/core/dataset/type';
-import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
-import { getPreviewChunks } from '@/web/core/dataset/api';
-import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
-import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
-import { useContextSelector } from 'use-context-selector';
-import { DatasetImportContext } from '../Context';
-import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
-import { getPreviewSourceReadType } from '../utils';
-
-const PreviewChunks = ({
- previewSource,
- onClose
-}: {
- previewSource: ImportSourceItemType;
- onClose: () => void;
-}) => {
- const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector(
- DatasetImportContext,
- (v) => v
- );
- const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
-
- const { data = [], loading: isLoading } = useRequest2(
- async () => {
- if (importSource === ImportDataSourceEnum.fileCustom) {
- const customSplitChar = processParamsForm.getValues('customSplitChar');
- const { chunks } = splitText2Chunks({
- text: previewSource.rawText || '',
- chunkLen: chunkSize,
- overlapRatio: chunkOverlapRatio,
- customReg: customSplitChar ? [customSplitChar] : []
- });
- return chunks.map((chunk) => ({
- q: chunk,
- a: ''
- }));
- }
-
- return getPreviewChunks({
- datasetId,
- type: getPreviewSourceReadType(previewSource),
- sourceId:
- previewSource.dbFileId ||
- previewSource.link ||
- previewSource.externalFileUrl ||
- previewSource.apiFileId ||
- '',
-
- chunkSize,
- overlapRatio: chunkOverlapRatio,
- customSplitChar: processParamsForm.getValues('customSplitChar'),
-
- selector: processParamsForm.getValues('webSelector'),
- isQAImport: importSource === ImportDataSourceEnum.csvTable,
- externalFileId: previewSource.externalFileId
- });
- },
- {
- manual: false
- }
- );
-
- return (
-
-
- {data.map((item, index) => (
-
- {item.q}
- {item.a}
-
- ))}
-
-
- );
-};
-
-export default React.memo(PreviewChunks);
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
index 068e0ea64..23f28ced3 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
@@ -8,10 +8,11 @@ import { useRouter } from 'next/router';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getDatasetCollectionById } from '@/web/core/dataset/api';
import MyBox from '@fastgpt/web/components/common/MyBox';
-import { ChunkSettingModeEnum } from '@/web/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
-import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { Box } from '@chakra-ui/react';
+import { DataChunkSplitModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
@@ -23,7 +24,6 @@ const ReTraining = () => {
collectionId: string;
};
- const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
const processParamsForm = useContextSelector(DatasetImportContext, (v) => v.processParamsForm);
@@ -46,18 +46,21 @@ const ReTraining = () => {
uploadedFileRate: 100
}
]);
+
processParamsForm.reset({
customPdfParse: collection.customPdfParse,
trainingType: collection.trainingType,
imageIndex: collection.imageIndex,
autoIndexes: collection.autoIndexes,
- chunkSettingMode: ChunkSettingModeEnum.auto,
+ chunkSettingMode: collection.chunkSettingMode || ChunkSettingModeEnum.auto,
+ chunkSplitMode: collection.chunkSplitMode || DataChunkSplitModeEnum.size,
embeddingChunkSize: collection.chunkSize,
qaChunkSize: collection.chunkSize,
- customSplitChar: collection.chunkSplitter,
- qaPrompt: collection.qaPrompt,
- webSelector: collection.metadata?.webPageSelector
+ indexSize: collection.indexSize || 512,
+ chunkSplitter: collection.chunkSplitter,
+ webSelector: collection.metadata?.webPageSelector,
+ qaPrompt: collection.qaPrompt || Prompt_AgentQA.description
});
}
});
diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx
index ff650630b..963177a00 100644
--- a/projects/app/src/pages/account/info/index.tsx
+++ b/projects/app/src/pages/account/info/index.tsx
@@ -294,7 +294,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
title={t('account_info:click_modify_nickname')}
borderColor={'transparent'}
transform={'translateX(-11px)'}
- maxLength={20}
+ maxLength={100}
onBlur={async (e) => {
const val = e.target.value;
if (val === userInfo?.team?.memberName) return;
diff --git a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts
index 0a5353d3c..919014a31 100644
--- a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts
@@ -2,8 +2,7 @@ import { reTrainingDatasetFileCollectionParams } from '@fastgpt/global/core/data
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
import {
DatasetCollectionTypeEnum,
- DatasetSourceReadTypeEnum,
- TrainingModeEnum
+ DatasetSourceReadTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { hashStr } from '@fastgpt/global/common/string/tools';
diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts
index e99e51626..f1b4f0ac5 100644
--- a/projects/app/src/pages/api/core/dataset/data/insertData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts
@@ -4,7 +4,7 @@
*/
import type { NextApiRequest } from 'next';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken/index';
-import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
+import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
@@ -16,6 +16,7 @@ import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils';
async function handler(req: NextApiRequest) {
const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps;
@@ -45,7 +46,7 @@ async function handler(req: NextApiRequest) {
// auth collection and get dataset
const [
{
- dataset: { _id: datasetId, vectorModel }
+ dataset: { _id: datasetId, vectorModel, agentModel }
}
] = await Promise.all([getCollectionWithDataset(collectionId)]);
@@ -60,9 +61,11 @@ async function handler(req: NextApiRequest) {
// token check
const token = await countPromptTokens(formatQ + formatA, '');
const vectorModelData = getEmbeddingModel(vectorModel);
+ const llmModelData = getLLMModel(agentModel);
+ const maxChunkSize = getLLMMaxChunkSize(llmModelData);
- if (token > vectorModelData.maxToken) {
- return Promise.reject('Q Over Tokens');
+ if (token > maxChunkSize) {
+ return Promise.reject(`Content over max chunk size: ${maxChunkSize}`);
}
// Duplicate data check
@@ -82,7 +85,7 @@ async function handler(req: NextApiRequest) {
q: formatQ,
a: formatA,
chunkIndex: 0,
- model: vectorModelData.model,
+ embeddingModel: vectorModelData.model,
indexes: formatIndexes
});
diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
index 71ade6289..caba7d922 100644
--- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
+++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
@@ -1,4 +1,9 @@
-import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import {
+ ChunkSettingModeEnum,
+ DataChunkSplitModeEnum,
+ DatasetCollectionDataProcessModeEnum,
+ DatasetSourceReadTypeEnum
+} from '@fastgpt/global/core/dataset/constants';
import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -8,17 +13,30 @@ import {
} from '@fastgpt/global/support/permission/constant';
import { authCollectionFile } from '@fastgpt/service/support/permission/auth/file';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
+import {
+ computeChunkSize,
+ computeChunkSplitter,
+ getLLMMaxChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
+import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { getLLMModel } from '@fastgpt/service/core/ai/model';
export type PostPreviewFilesChunksProps = {
datasetId: string;
type: DatasetSourceReadTypeEnum;
sourceId: string;
- chunkSize: number;
- overlapRatio: number;
- customSplitChar?: string;
customPdfParse?: boolean;
+ trainingType: DatasetCollectionDataProcessModeEnum;
+
+ // Chunk settings
+ chunkSettingMode: ChunkSettingModeEnum;
+ chunkSplitMode: DataChunkSplitModeEnum;
+ chunkSize: number;
+ chunkSplitter?: string;
+ overlapRatio: number;
+
// Read params
selector?: string;
isQAImport?: boolean;
@@ -32,55 +50,64 @@ export type PreviewChunksResponse = {
async function handler(
req: ApiRequestProps
): Promise {
- const {
+ let {
type,
sourceId,
+ customPdfParse = false,
+
+ trainingType,
+ chunkSettingMode,
+ chunkSplitMode,
chunkSize,
- customSplitChar,
+ chunkSplitter,
+
overlapRatio,
selector,
isQAImport,
datasetId,
- externalFileId,
- customPdfParse = false
+ externalFileId
} = req.body;
if (!sourceId) {
throw new Error('sourceId is empty');
}
- if (chunkSize > 30000) {
- throw new Error('chunkSize is too large, should be less than 30000');
+
+ const fileAuthRes =
+ type === DatasetSourceReadTypeEnum.fileLocal
+ ? await authCollectionFile({
+ req,
+ authToken: true,
+ authApiKey: true,
+ fileId: sourceId,
+ per: OwnerPermissionVal
+ })
+ : undefined;
+
+ const { dataset, teamId, tmbId } = await authDataset({
+ req,
+ authApiKey: true,
+ authToken: true,
+ datasetId,
+ per: WritePermissionVal
+ });
+
+ if (fileAuthRes && (String(fileAuthRes.tmbId) !== String(tmbId) || !fileAuthRes.isRoot)) {
+ return Promise.reject(CommonErrEnum.unAuthFile);
}
- const { teamId, tmbId, apiServer, feishuServer, yuqueServer } = await (async () => {
- if (type === DatasetSourceReadTypeEnum.fileLocal) {
- const res = await authCollectionFile({
- req,
- authToken: true,
- authApiKey: true,
- fileId: sourceId,
- per: OwnerPermissionVal
- });
- return {
- teamId: res.teamId,
- tmbId: res.tmbId
- };
- }
- const { dataset, teamId, tmbId } = await authDataset({
- req,
- authApiKey: true,
- authToken: true,
- datasetId,
- per: WritePermissionVal
- });
- return {
- teamId,
- tmbId,
- apiServer: dataset.apiServer,
- feishuServer: dataset.feishuServer,
- yuqueServer: dataset.yuqueServer
- };
- })();
+ chunkSize = computeChunkSize({
+ trainingType,
+ chunkSettingMode,
+ chunkSplitMode,
+ chunkSize,
+ llmModel: getLLMModel(dataset.agentModel)
+ });
+
+ chunkSplitter = computeChunkSplitter({
+ chunkSettingMode,
+ chunkSplitMode,
+ chunkSplitter
+ });
const { rawText } = await readDatasetSourceRawText({
teamId,
@@ -89,18 +116,19 @@ async function handler(
sourceId,
selector,
isQAImport,
- apiServer,
- feishuServer,
- yuqueServer,
+ apiServer: dataset.apiServer,
+ feishuServer: dataset.feishuServer,
+ yuqueServer: dataset.yuqueServer,
externalFileId,
customPdfParse
});
return rawText2Chunks({
rawText,
- chunkLen: chunkSize,
+ chunkSize,
+ maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
overlapRatio,
- customReg: customSplitChar ? [customSplitChar] : [],
+ customReg: chunkSplitter ? [chunkSplitter] : [],
isQAImport: isQAImport
}).slice(0, 10);
}
diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts
index b5132e2de..3867b56e1 100644
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -5,25 +5,63 @@ import {
UpdateDatasetDataProps
} from '@fastgpt/global/core/dataset/controller';
import { insertDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
-import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
import { DatasetDataIndexItemType, DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
-import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
+import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { ClientSession } from '@fastgpt/service/common/mongo';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
+import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
+import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils';
-const formatIndexes = ({
+const formatIndexes = async ({
indexes,
q,
- a = ''
+ a = '',
+ indexSize
}: {
indexes?: (Omit & { dataId?: string })[];
q: string;
a?: string;
-}) => {
+ indexSize: number;
+}): Promise<
+ {
+ type: `${DatasetDataIndexTypeEnum}`;
+ text: string;
+ dataId?: string;
+ }[]
+> => {
+ /* get dataset data default index */
+ const getDefaultIndex = ({
+ q = '',
+ a,
+ indexSize
+ }: {
+ q?: string;
+ a?: string;
+ indexSize: number;
+ }) => {
+ const qChunks = splitText2Chunks({
+ text: q,
+ chunkSize: indexSize
+ }).chunks;
+ const aChunks = a ? splitText2Chunks({ text: a, chunkSize: indexSize }).chunks : [];
+
+ return [
+ ...qChunks.map((text) => ({
+ text,
+ type: DatasetDataIndexTypeEnum.default
+ })),
+ ...aChunks.map((text) => ({
+ text,
+ type: DatasetDataIndexTypeEnum.default
+ }))
+ ];
+ };
+
indexes = indexes || [];
// If index not type, set it to custom
indexes = indexes
@@ -35,7 +73,7 @@ const formatIndexes = ({
.filter((item) => !!item.text.trim());
// Recompute default indexes, Merge ids of the same index, reduce the number of rebuilds
- const defaultIndexes = getDefaultIndex({ q, a });
+ const defaultIndexes = getDefaultIndex({ q, a, indexSize });
const concatDefaultIndexes = defaultIndexes.map((item) => {
const oldIndex = indexes!.find((index) => index.text === item.text);
if (oldIndex) {
@@ -56,11 +94,24 @@ const formatIndexes = ({
(item, index, self) => index === self.findIndex((t) => t.text === item.text)
);
- return indexes.map((index) => ({
- type: index.type,
- text: index.text,
- dataId: index.dataId
- }));
+ const chekcIndexes = (
+ await Promise.all(
+ indexes.map(async (item) => {
+ // If oversize tokens, split it
+ const tokens = await countPromptTokens(item.text);
+ if (tokens > indexSize) {
+ const splitText = splitText2Chunks({ text: item.text, chunkSize: 512 }).chunks;
+ return splitText.map((text) => ({
+ text,
+ type: item.type
+ }));
+ }
+ return item;
+ })
+ )
+ ).flat();
+
+ return chekcIndexes;
};
/* insert data.
* 1. create data id
@@ -75,30 +126,40 @@ export async function insertData2Dataset({
q,
a = '',
chunkIndex = 0,
+ indexSize = 512,
indexes,
- model,
+ embeddingModel,
session
}: CreateDatasetDataProps & {
- model: string;
+ embeddingModel: string;
+ indexSize?: number;
session?: ClientSession;
}) {
- if (!q || !datasetId || !collectionId || !model) {
- return Promise.reject('q, datasetId, collectionId, model is required');
+ if (!q || !datasetId || !collectionId || !embeddingModel) {
+ return Promise.reject('q, datasetId, collectionId, embeddingModel is required');
}
if (String(teamId) === String(tmbId)) {
return Promise.reject("teamId and tmbId can't be the same");
}
+ const embModel = getEmbeddingModel(embeddingModel);
+ indexSize = Math.min(embModel.maxToken, indexSize);
+
// 1. Get vector indexes and insert
// Empty indexes check, if empty, create default index
- const newIndexes = formatIndexes({ indexes, q, a });
+ const newIndexes = await formatIndexes({
+ indexes,
+ q,
+ a,
+ indexSize
+ });
// insert to vector store
const result = await Promise.all(
newIndexes.map(async (item) => {
const result = await insertDatasetDataVector({
query: item.text,
- model: getEmbeddingModel(model),
+ model: embModel,
teamId,
datasetId,
collectionId
@@ -163,8 +224,9 @@ export async function updateData2Dataset({
q = '',
a,
indexes,
- model
-}: UpdateDatasetDataProps & { model: string }) {
+ model,
+ indexSize = 512
+}: UpdateDatasetDataProps & { model: string; indexSize?: number }) {
if (!Array.isArray(indexes)) {
return Promise.reject('indexes is required');
}
@@ -174,7 +236,7 @@ export async function updateData2Dataset({
if (!mongoData) return Promise.reject('core.dataset.error.Data not found');
// 2. Compute indexes
- const formatIndexesResult = formatIndexes({ indexes, q, a });
+ const formatIndexesResult = await formatIndexes({ indexes, q, a, indexSize });
// 3. Patch indexes, create, update, delete
const patchResult: PatchIndexesProps[] = [];
diff --git a/projects/app/src/service/events/generateQA.ts b/projects/app/src/service/events/generateQA.ts
index 4a335cc22..2273d1edf 100644
--- a/projects/app/src/service/events/generateQA.ts
+++ b/projects/app/src/service/events/generateQA.ts
@@ -21,6 +21,11 @@ import {
llmCompletionsBodyFormat,
llmStreamResponseToAnswerText
} from '@fastgpt/service/core/ai/utils';
+import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
+import {
+ chunkAutoChunkSize,
+ getLLMMaxChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
const reduceQueue = () => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -129,7 +134,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
});
const answer = await llmStreamResponseToAnswerText(chatResponse);
- const qaArr = formatSplitText(answer, text); // 格式化后的QA对
+ const qaArr = formatSplitText({ answer, rawText: text, llmModel: modelData }); // 格式化后的QA对
addLog.info(`[QA Queue] Finish`, {
time: Date.now() - startTime,
@@ -180,10 +185,18 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
}
// Format qa answer
-function formatSplitText(text: string, rawText: string) {
- text = text.replace(/\\n/g, '\n'); // 将换行符替换为空格
+function formatSplitText({
+ answer,
+ rawText,
+ llmModel
+}: {
+ answer: string;
+ rawText: string;
+ llmModel: LLMModelItemType;
+}) {
+ answer = answer.replace(/\\n/g, '\n'); // 将换行符替换为空格
const regex = /Q\d+:(\s*)(.*)(\s*)A\d+:(\s*)([\s\S]*?)(?=Q\d|$)/g; // 匹配Q和A的正则表达式
- const matches = text.matchAll(regex); // 获取所有匹配到的结果
+ const matches = answer.matchAll(regex); // 获取所有匹配到的结果
const result: PushDatasetDataChunkProps[] = []; // 存储最终的结果
for (const match of matches) {
@@ -199,7 +212,11 @@ function formatSplitText(text: string, rawText: string) {
// empty result. direct split chunk
if (result.length === 0) {
- const { chunks } = splitText2Chunks({ text: rawText, chunkLen: 512 });
+ const { chunks } = splitText2Chunks({
+ text: rawText,
+ chunkSize: chunkAutoChunkSize,
+ maxSize: getLLMMaxChunkSize(llmModel)
+ });
chunks.forEach((chunk) => {
result.push({
q: chunk,
diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts
index a8e4f5ca9..0e7f8c2b4 100644
--- a/projects/app/src/service/events/generateVector.ts
+++ b/projects/app/src/service/events/generateVector.ts
@@ -245,7 +245,7 @@ const insertData = async ({
a: trainingData.a,
chunkIndex: trainingData.chunkIndex,
indexes: trainingData.indexes,
- model: trainingData.model,
+ embeddingModel: trainingData.model,
session
});
// delete data from training
diff --git a/projects/app/src/web/core/dataset/constants.ts b/projects/app/src/web/core/dataset/constants.ts
index 860e73502..3976d93ee 100644
--- a/projects/app/src/web/core/dataset/constants.ts
+++ b/projects/app/src/web/core/dataset/constants.ts
@@ -60,15 +60,11 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
createTime: new Date(),
trainingType: DatasetCollectionDataProcessModeEnum.chunk,
chunkSize: 0,
+ indexSize: 512,
permission: new DatasetPermission(),
indexAmount: 0
};
-export enum ChunkSettingModeEnum {
- auto = 'auto',
- custom = 'custom'
-}
-
export const datasetTypeCourseMap: Record<`${DatasetTypeEnum}`, string> = {
[DatasetTypeEnum.folder]: '',
[DatasetTypeEnum.dataset]: '',
diff --git a/projects/app/src/web/core/dataset/type.d.ts b/projects/app/src/web/core/dataset/type.d.ts
index a095bc798..0f69d9656 100644
--- a/projects/app/src/web/core/dataset/type.d.ts
+++ b/projects/app/src/web/core/dataset/type.d.ts
@@ -1,6 +1,6 @@
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
-import { ChunkSettingModeEnum } from './constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { UseFormReturn } from 'react-hook-form';
import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
@@ -41,7 +41,7 @@ export type ImportSourceParamsType = UseFormReturn<
{
chunkSize: number;
chunkOverlapRatio: number;
- customSplitChar: string;
+ chunkSplitter: string;
prompt: string;
mode: TrainingModeEnum;
way: ChunkSettingModeEnum;
From a680b565ea4e66f7fcc20b07220f0df8e2d3b5fc Mon Sep 17 00:00:00 2001
From: heheer
Date: Fri, 21 Mar 2025 17:19:06 +0800
Subject: [PATCH 14/35] fix input form label overflow (#4266)
---
.../src/components/core/chat/components/AIResponseBox.tsx | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/projects/app/src/components/core/chat/components/AIResponseBox.tsx b/projects/app/src/components/core/chat/components/AIResponseBox.tsx
index 81248a6d6..9131631aa 100644
--- a/projects/app/src/components/core/chat/components/AIResponseBox.tsx
+++ b/projects/app/src/components/core/chat/components/AIResponseBox.tsx
@@ -268,9 +268,11 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({
{interactive.params.description && }
{interactive.params.inputForm?.map((input) => (
-
- {input.label}
- {input.description && }
+
+
+ {input.label}
+ {input.description && }
+
{input.type === FlowNodeInputTypeEnum.input && (
Date: Mon, 24 Mar 2025 13:49:43 +0800
Subject: [PATCH 15/35] add model test log (#4272)
* sync collection
* remove lock
* add model test log
* update ui
* update log
* fix: channel test
* preview chunk ui
* test model ux
* test model log
* perf: dataset selector
* fix: system plugin auth
* update nextjs
---
.../zh-cn/docs/development/upgrading/492.md | 6 +-
packages/global/core/plugin/type.d.ts | 2 +
packages/global/package.json | 2 +-
.../service/core/ai/audio/transcriptions.ts | 12 +-
.../service/core/app/plugin/controller.ts | 23 +++-
.../core/workflow/dispatch/plugin/run.ts | 4 +-
packages/service/package.json | 2 +-
packages/web/i18n/en/account_model.json | 1 +
packages/web/i18n/en/dataset.json | 2 +-
packages/web/i18n/zh-CN/account_model.json | 1 +
packages/web/i18n/zh-CN/dataset.json | 2 +-
packages/web/i18n/zh-Hant/account_model.json | 1 +
packages/web/i18n/zh-Hant/dataset.json | 2 +-
pnpm-lock.yaml | 122 +++++++++---------
projects/app/package.json | 2 +-
.../core/app/DatasetSelectModal.tsx | 66 ++++++----
.../core/chat/components/AIResponseBox.tsx | 10 +-
.../chat/components/WholeResponseModal.tsx | 32 ++---
.../account/model/Channel/index.tsx | 14 +-
.../account/model/Log/index.tsx | 4 +-
.../Import/commonProgress/PreviewData.tsx | 21 +--
.../app/src/pages/api/core/ai/model/test.ts | 13 +-
.../api/core/dataset/file/getPreviewChunks.ts | 17 ++-
.../src/pages/api/v1/audio/transcriptions.ts | 2 +-
24 files changed, 210 insertions(+), 153 deletions(-)
diff --git a/docSite/content/zh-cn/docs/development/upgrading/492.md b/docSite/content/zh-cn/docs/development/upgrading/492.md
index 73d156129..96f8b5304 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
@@ -25,7 +25,11 @@ weight: 799
2. 邀请链接交互。
3. 无 SSL 证书时复制失败,会提示弹窗用于手动复制。
4. FastGPT 未内置 ai proxy 渠道时,也能正常展示其名称。
+5. 升级 nextjs 版本至 14.2.25。
## 🐛 修复
-1. 飞书和语雀知识库无法同步。
\ No newline at end of file
+1. 飞书和语雀知识库无法同步。
+2. 渠道测试时,如果配置了模型自定义请求地址,会走自定义请求地址,而不是渠道请求地址。
+3. 语音识别模型测试未启用的模型时,无法正常测试。
+4. 管理员配置系统插件时,如果插件包含其他系统应用,无法正常鉴权。
\ No newline at end of file
diff --git a/packages/global/core/plugin/type.d.ts b/packages/global/core/plugin/type.d.ts
index 925e4549b..b38930ffe 100644
--- a/packages/global/core/plugin/type.d.ts
+++ b/packages/global/core/plugin/type.d.ts
@@ -41,6 +41,8 @@ export type PluginTemplateType = PluginRuntimeType & {
export type PluginRuntimeType = {
id: string;
teamId?: string;
+ tmbId?: string;
+
name: string;
avatar: string;
showStatus?: boolean;
diff --git a/packages/global/package.json b/packages/global/package.json
index 70f7f1db4..c71c9d445 100644
--- a/packages/global/package.json
+++ b/packages/global/package.json
@@ -10,7 +10,7 @@
"js-yaml": "^4.1.0",
"jschardet": "3.1.1",
"nanoid": "^5.1.3",
- "next": "14.2.24",
+ "next": "14.2.25",
"openai": "4.61.0",
"openapi-types": "^12.1.3",
"json5": "^2.2.3",
diff --git a/packages/service/core/ai/audio/transcriptions.ts b/packages/service/core/ai/audio/transcriptions.ts
index 9d93ede9f..4cf7dc70f 100644
--- a/packages/service/core/ai/audio/transcriptions.ts
+++ b/packages/service/core/ai/audio/transcriptions.ts
@@ -3,21 +3,25 @@ import { getAxiosConfig } from '../config';
import axios from 'axios';
import FormData from 'form-data';
import { getSTTModel } from '../model';
+import { STTModelType } from '@fastgpt/global/core/ai/model.d';
export const aiTranscriptions = async ({
- model,
+ model: modelData,
fileStream,
headers
}: {
- model: string;
+ model: STTModelType;
fileStream: fs.ReadStream;
headers?: Record;
}) => {
+ if (!modelData) {
+ return Promise.reject('no model');
+ }
+
const data = new FormData();
- data.append('model', model);
+ data.append('model', modelData.model);
data.append('file', fileStream);
- const modelData = getSTTModel(model);
const aiAxiosConfig = getAxiosConfig();
const { data: result } = await axios<{ text: string }>({
diff --git a/packages/service/core/app/plugin/controller.ts b/packages/service/core/app/plugin/controller.ts
index 2d185686d..dbbb7e273 100644
--- a/packages/service/core/app/plugin/controller.ts
+++ b/packages/service/core/app/plugin/controller.ts
@@ -37,11 +37,12 @@ export async function splitCombinePluginId(id: string) {
return { source, pluginId: id };
}
-type ChildAppType = SystemPluginTemplateItemType & { teamId?: string };
+type ChildAppType = SystemPluginTemplateItemType & { teamId?: string; tmbId?: string };
+
const getSystemPluginTemplateById = async (
pluginId: string,
versionId?: string
-): Promise => {
+): Promise => {
const item = getSystemPluginTemplates().find((plugin) => plugin.id === pluginId);
if (!item) return Promise.reject(PluginErrEnum.unAuth);
@@ -67,12 +68,17 @@ const getSystemPluginTemplateById = async (
: await getAppLatestVersion(plugin.associatedPluginId, app);
if (!version.versionId) return Promise.reject('App version not found');
- plugin.workflow = {
- nodes: version.nodes,
- edges: version.edges,
- chatConfig: version.chatConfig
+ return {
+ ...plugin,
+ workflow: {
+ nodes: version.nodes,
+ edges: version.edges,
+ chatConfig: version.chatConfig
+ },
+ version: versionId || String(version.versionId),
+ teamId: String(app.teamId),
+ tmbId: String(app.tmbId)
};
- plugin.version = versionId || String(version.versionId);
}
return plugin;
};
@@ -168,6 +174,7 @@ export async function getChildAppRuntimeById(
return {
id: String(item._id),
teamId: String(item.teamId),
+ tmbId: String(item.tmbId),
name: item.name,
avatar: item.avatar,
intro: item.intro,
@@ -187,6 +194,7 @@ export async function getChildAppRuntimeById(
pluginOrder: 0
};
} else {
+ // System
return getSystemPluginTemplateById(pluginId, versionId);
}
})();
@@ -194,6 +202,7 @@ export async function getChildAppRuntimeById(
return {
id: app.id,
teamId: app.teamId,
+ tmbId: app.tmbId,
name: app.name,
avatar: app.avatar,
showStatus: app.showStatus,
diff --git a/packages/service/core/workflow/dispatch/plugin/run.ts b/packages/service/core/workflow/dispatch/plugin/run.ts
index a81a6a867..b50995200 100644
--- a/packages/service/core/workflow/dispatch/plugin/run.ts
+++ b/packages/service/core/workflow/dispatch/plugin/run.ts
@@ -88,9 +88,9 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise= 10'}
cpu: [arm64]
os: [darwin]
- '@next/swc-darwin-x64@14.2.24':
- resolution: {integrity: sha512-lXR2WQqUtu69l5JMdTwSvQUkdqAhEWOqJEYUQ21QczQsAlNOW2kWZCucA6b3EXmPbcvmHB1kSZDua/713d52xg==}
+ '@next/swc-darwin-x64@14.2.25':
+ resolution: {integrity: sha512-V+iYM/QR+aYeJl3/FWWU/7Ix4b07ovsQ5IbkwgUK29pTHmq+5UxeDr7/dphvtXEq5pLB/PucfcBNh9KZ8vWbug==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
- '@next/swc-linux-arm64-gnu@14.2.24':
- resolution: {integrity: sha512-nxvJgWOpSNmzidYvvGDfXwxkijb6hL9+cjZx1PVG6urr2h2jUqBALkKjT7kpfurRWicK6hFOvarmaWsINT1hnA==}
+ '@next/swc-linux-arm64-gnu@14.2.25':
+ resolution: {integrity: sha512-LFnV2899PJZAIEHQ4IMmZIgL0FBieh5keMnriMY1cK7ompR+JUd24xeTtKkcaw8QmxmEdhoE5Mu9dPSuDBgtTg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
- '@next/swc-linux-arm64-musl@14.2.24':
- resolution: {integrity: sha512-PaBgOPhqa4Abxa3y/P92F3kklNPsiFjcjldQGT7kFmiY5nuFn8ClBEoX8GIpqU1ODP2y8P6hio6vTomx2Vy0UQ==}
+ '@next/swc-linux-arm64-musl@14.2.25':
+ resolution: {integrity: sha512-QC5y5PPTmtqFExcKWKYgUNkHeHE/z3lUsu83di488nyP0ZzQ3Yse2G6TCxz6nNsQwgAx1BehAJTZez+UQxzLfw==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
- '@next/swc-linux-x64-gnu@14.2.24':
- resolution: {integrity: sha512-vEbyadiRI7GOr94hd2AB15LFVgcJZQWu7Cdi9cWjCMeCiUsHWA0U5BkGPuoYRnTxTn0HacuMb9NeAmStfBCLoQ==}
+ '@next/swc-linux-x64-gnu@14.2.25':
+ resolution: {integrity: sha512-y6/ML4b9eQ2D/56wqatTJN5/JR8/xdObU2Fb1RBidnrr450HLCKr6IJZbPqbv7NXmje61UyxjF5kvSajvjye5w==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
- '@next/swc-linux-x64-musl@14.2.24':
- resolution: {integrity: sha512-df0FC9ptaYsd8nQCINCzFtDWtko8PNRTAU0/+d7hy47E0oC17tI54U/0NdGk7l/76jz1J377dvRjmt6IUdkpzQ==}
+ '@next/swc-linux-x64-musl@14.2.25':
+ resolution: {integrity: sha512-sPX0TSXHGUOZFvv96GoBXpB3w4emMqKeMgemrSxI7A6l55VBJp/RKYLwZIB9JxSqYPApqiREaIIap+wWq0RU8w==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
- '@next/swc-win32-arm64-msvc@14.2.24':
- resolution: {integrity: sha512-ZEntbLjeYAJ286eAqbxpZHhDFYpYjArotQ+/TW9j7UROh0DUmX7wYDGtsTPpfCV8V+UoqHBPU7q9D4nDNH014Q==}
+ '@next/swc-win32-arm64-msvc@14.2.25':
+ resolution: {integrity: sha512-ReO9S5hkA1DU2cFCsGoOEp7WJkhFzNbU/3VUF6XxNGUCQChyug6hZdYL/istQgfT/GWE6PNIg9cm784OI4ddxQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
- '@next/swc-win32-ia32-msvc@14.2.24':
- resolution: {integrity: sha512-9KuS+XUXM3T6v7leeWU0erpJ6NsFIwiTFD5nzNg8J5uo/DMIPvCp3L1Ao5HjbHX0gkWPB1VrKoo/Il4F0cGK2Q==}
+ '@next/swc-win32-ia32-msvc@14.2.25':
+ resolution: {integrity: sha512-DZ/gc0o9neuCDyD5IumyTGHVun2dCox5TfPQI/BJTYwpSNYM3CZDI4i6TOdjeq1JMo+Ug4kPSMuZdwsycwFbAw==}
engines: {node: '>= 10'}
cpu: [ia32]
os: [win32]
- '@next/swc-win32-x64-msvc@14.2.24':
- resolution: {integrity: sha512-cXcJ2+x0fXQ2CntaE00d7uUH+u1Bfp/E0HsNQH79YiLaZE5Rbm7dZzyAYccn3uICM7mw+DxoMqEfGXZtF4Fgaw==}
+ '@next/swc-win32-x64-msvc@14.2.25':
+ resolution: {integrity: sha512-KSznmS6eFjQ9RJ1nEc66kJvtGIL1iZMYmGEXsZPh2YtnLtqrgdVvKXJY2ScjjoFnG6nGLyPFR0UiEvDwVah4Tw==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
@@ -7191,8 +7191,8 @@ packages:
react: '>= 17.0.2'
react-i18next: '>= 13.5.0'
- next@14.2.24:
- resolution: {integrity: sha512-En8VEexSJ0Py2FfVnRRh8gtERwDRaJGNvsvad47ShkC2Yi8AXQPXEA2vKoDJlGFSj5WE5SyF21zNi4M5gyi+SQ==}
+ next@14.2.25:
+ resolution: {integrity: sha512-N5M7xMc4wSb4IkPvEV5X2BRRXUmhVHNyaXwEM86+voXthSZz8ZiRyQW4p9mwAoAPIm6OzuVZtn7idgEJeAJN3Q==}
engines: {node: '>=18.17.0'}
hasBin: true
peerDependencies:
@@ -10652,12 +10652,12 @@ snapshots:
'@chakra-ui/system': 2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1)
react: 18.3.1
- '@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
+ '@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
dependencies:
'@chakra-ui/react': 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@emotion/cache': 11.14.0
'@emotion/react': 11.11.1(@types/react@18.3.1)(react@18.3.1)
- next: 14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
+ next: 14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
react: 18.3.1
'@chakra-ui/object-utils@2.1.0': {}
@@ -11665,37 +11665,37 @@ snapshots:
'@nestjs/core': 10.4.15(@nestjs/common@10.4.15(reflect-metadata@0.2.2)(rxjs@7.8.2))(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.2)
tslib: 2.8.1
- '@next/env@14.2.24': {}
+ '@next/env@14.2.25': {}
'@next/eslint-plugin-next@14.2.24':
dependencies:
glob: 10.3.10
- '@next/swc-darwin-arm64@14.2.24':
+ '@next/swc-darwin-arm64@14.2.25':
optional: true
- '@next/swc-darwin-x64@14.2.24':
+ '@next/swc-darwin-x64@14.2.25':
optional: true
- '@next/swc-linux-arm64-gnu@14.2.24':
+ '@next/swc-linux-arm64-gnu@14.2.25':
optional: true
- '@next/swc-linux-arm64-musl@14.2.24':
+ '@next/swc-linux-arm64-musl@14.2.25':
optional: true
- '@next/swc-linux-x64-gnu@14.2.24':
+ '@next/swc-linux-x64-gnu@14.2.25':
optional: true
- '@next/swc-linux-x64-musl@14.2.24':
+ '@next/swc-linux-x64-musl@14.2.25':
optional: true
- '@next/swc-win32-arm64-msvc@14.2.24':
+ '@next/swc-win32-arm64-msvc@14.2.25':
optional: true
- '@next/swc-win32-ia32-msvc@14.2.24':
+ '@next/swc-win32-ia32-msvc@14.2.25':
optional: true
- '@next/swc-win32-x64-msvc@14.2.24':
+ '@next/swc-win32-x64-msvc@14.2.25':
optional: true
'@node-rs/jieba-android-arm-eabi@2.0.1':
@@ -17532,7 +17532,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
- next-i18next@15.4.2(i18next@23.16.8)(next@14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
+ next-i18next@15.4.2(i18next@23.16.8)(next@14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
dependencies:
'@babel/runtime': 7.26.10
'@types/hoist-non-react-statics': 3.3.6
@@ -17540,13 +17540,13 @@ snapshots:
hoist-non-react-statics: 3.3.2
i18next: 23.16.8
i18next-fs-backend: 2.6.0
- next: 14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
+ next: 14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
react: 18.3.1
react-i18next: 14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
- next@14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1):
+ next@14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1):
dependencies:
- '@next/env': 14.2.24
+ '@next/env': 14.2.25
'@swc/helpers': 0.5.5
busboy: 1.6.0
caniuse-lite: 1.0.30001704
@@ -17556,24 +17556,24 @@ snapshots:
react-dom: 18.3.1(react@18.3.1)
styled-jsx: 5.1.1(@babel/core@7.26.10)(react@18.3.1)
optionalDependencies:
- '@next/swc-darwin-arm64': 14.2.24
- '@next/swc-darwin-x64': 14.2.24
- '@next/swc-linux-arm64-gnu': 14.2.24
- '@next/swc-linux-arm64-musl': 14.2.24
- '@next/swc-linux-x64-gnu': 14.2.24
- '@next/swc-linux-x64-musl': 14.2.24
- '@next/swc-win32-arm64-msvc': 14.2.24
- '@next/swc-win32-ia32-msvc': 14.2.24
- '@next/swc-win32-x64-msvc': 14.2.24
+ '@next/swc-darwin-arm64': 14.2.25
+ '@next/swc-darwin-x64': 14.2.25
+ '@next/swc-linux-arm64-gnu': 14.2.25
+ '@next/swc-linux-arm64-musl': 14.2.25
+ '@next/swc-linux-x64-gnu': 14.2.25
+ '@next/swc-linux-x64-musl': 14.2.25
+ '@next/swc-win32-arm64-msvc': 14.2.25
+ '@next/swc-win32-ia32-msvc': 14.2.25
+ '@next/swc-win32-x64-msvc': 14.2.25
sass: 1.85.1
transitivePeerDependencies:
- '@babel/core'
- babel-plugin-macros
- nextjs-cors@2.2.0(next@14.2.24(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)):
+ nextjs-cors@2.2.0(next@14.2.25(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)):
dependencies:
cors: 2.8.5
- next: 14.2.24(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
+ next: 14.2.25(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
node-abi@3.74.0:
dependencies:
diff --git a/projects/app/package.json b/projects/app/package.json
index 60a43cdde..b37c1b5ad 100644
--- a/projects/app/package.json
+++ b/projects/app/package.json
@@ -42,7 +42,7 @@
"lodash": "^4.17.21",
"mermaid": "^10.2.3",
"nanoid": "^5.1.3",
- "next": "14.2.24",
+ "next": "14.2.25",
"next-i18next": "15.4.2",
"nprogress": "^0.2.0",
"qrcode": "^1.5.4",
diff --git a/projects/app/src/components/core/app/DatasetSelectModal.tsx b/projects/app/src/components/core/app/DatasetSelectModal.tsx
index 8ec04227a..7010ae218 100644
--- a/projects/app/src/components/core/app/DatasetSelectModal.tsx
+++ b/projects/app/src/components/core/app/DatasetSelectModal.tsx
@@ -69,31 +69,37 @@ export const DatasetSelectModal = ({
{selectedDatasets.map((item) =>
(() => {
return (
-
-
-
-
- {item.name}
-
- {
- setSelectedDatasets((state) =>
- state.filter((dataset) => dataset.datasetId !== item.datasetId)
- );
- }}
- />
-
-
+
+
+
+
+
+ {item.name}
+
+ {
+ setSelectedDatasets((state) =>
+ state.filter((dataset) => dataset.datasetId !== item.datasetId)
+ );
+ }}
+ />
+
+
+
);
})()
)}
@@ -117,7 +123,7 @@ export const DatasetSelectModal = ({
label={
item.type === DatasetTypeEnum.folder
? t('common:dataset.Select Folder')
- : t('common:dataset.Select Dataset')
+ : item.name
}
>
-
+
{item.name}
diff --git a/projects/app/src/components/core/chat/components/AIResponseBox.tsx b/projects/app/src/components/core/chat/components/AIResponseBox.tsx
index 9131631aa..45759061f 100644
--- a/projects/app/src/components/core/chat/components/AIResponseBox.tsx
+++ b/projects/app/src/components/core/chat/components/AIResponseBox.tsx
@@ -268,12 +268,10 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({
{interactive.params.description && }
{interactive.params.inputForm?.map((input) => (
-
-
- {input.label}
- {input.description && }
-
-
+
+ {input.label}
+ {input.description && }
+
{input.type === FlowNodeInputTypeEnum.input && (
-
- {activeModule?.searchUsingReRank ? (
- activeModule?.rerankModel ? (
- {`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}
+ {activeModule?.searchUsingReRank !== undefined && (
+
+ {activeModule?.searchUsingReRank ? (
+ activeModule?.rerankModel ? (
+ {`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}
+ ) : (
+ 'True'
+ )
) : (
- 'True'
- )
- ) : (
- `False`
- )}
-
- }
- />
+ `False`
+ )}
+
+ }
+ />
+ )}
{activeModule.queryExtensionResult && (
<>
import('./EditChannelModal'), { ssr: false });
const ModelTest = dynamic(() => import('./ModelTest'), { ssr: false });
@@ -77,6 +78,9 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
}
);
+ const { openConfirm, ConfirmModal } = useConfirm({
+ type: 'delete'
+ });
const { runAsync: onDeleteChannel, loading: loadingDeleteChannel } = useRequest2(deleteChannel, {
manual: true,
onSuccess: () => {
@@ -212,7 +216,14 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
type: 'danger',
icon: 'delete',
label: t('common:common.Delete'),
- onClick: () => onDeleteChannel(item.id)
+ onClick: () =>
+ openConfirm(
+ () => onDeleteChannel(item.id),
+ undefined,
+ t('account_model:confirm_delete_channel', {
+ name: item.name
+ })
+ )()
}
]
}
@@ -238,6 +249,7 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
{!!modelTestData && (
setTestModelData(undefined)} />
)}
+
>
);
};
diff --git a/projects/app/src/pageComponents/account/model/Log/index.tsx b/projects/app/src/pageComponents/account/model/Log/index.tsx
index afe811895..f57ad9d69 100644
--- a/projects/app/src/pageComponents/account/model/Log/index.tsx
+++ b/projects/app/src/pageComponents/account/model/Log/index.tsx
@@ -197,7 +197,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
/>
-
+
{t('account_model:channel_name')}
@@ -210,7 +210,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
/>
-
+
{t('account_model:model_name')}
diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
index 37a09bc2c..57eab40ee 100644
--- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
@@ -34,9 +34,9 @@ const PreviewData = () => {
const [previewFile, setPreviewFile] = useState();
- const { data = [], loading: isLoading } = useRequest2(
+ const { data = { chunks: [], total: 0 }, loading: isLoading } = useRequest2(
async () => {
- if (!previewFile) return;
+ if (!previewFile) return { chunks: [], total: 0 };
if (importSource === ImportDataSourceEnum.fileCustom) {
const chunkSplitter = processParamsForm.getValues('chunkSplitter');
const { chunks } = splitText2Chunks({
@@ -46,10 +46,13 @@ const PreviewData = () => {
overlapRatio: chunkOverlapRatio,
customReg: chunkSplitter ? [chunkSplitter] : []
});
- return chunks.map((chunk) => ({
- q: chunk,
- a: ''
- }));
+ return {
+ chunks: chunks.map((chunk) => ({
+ q: chunk,
+ a: ''
+ })),
+ total: chunks.length
+ };
}
return getPreviewChunks({
@@ -81,7 +84,7 @@ const PreviewData = () => {
manual: false,
onSuccess(result) {
if (!previewFile) return;
- if (!result || result.length === 0) {
+ if (!result || result.total === 0) {
toast({
title: t('dataset:preview_chunk_empty'),
status: 'error'
@@ -130,14 +133,14 @@ const PreviewData = () => {
{t('dataset:preview_chunk')}
- {t('dataset:preview_chunk_intro')}
+ {t('dataset:preview_chunk_intro', { total: data.total })}
{previewFile ? (
<>
- {data.map((item, index) => (
+ {data.chunks.map((item, index) => (
= channelId
? {
'Aiproxy-Channel': String(channelId)
}
: {};
+ addLog.debug(`Test model`, modelData);
if (modelData.type === 'llm') {
return testLLMModel(modelData, headers);
@@ -63,10 +69,6 @@ async function handler(
export default NextAPI(handler);
const testLLMModel = async (model: LLMModelItemType, headers: Record) => {
- const ai = getAIApi({
- timeout: 10000
- });
-
const requestBody = llmCompletionsBodyFormat(
{
model: model.model,
@@ -75,6 +77,7 @@ const testLLMModel = async (model: LLMModelItemType, headers: Record
const testSTTModel = async (model: STTModelType, headers: Record) => {
const path = isProduction ? '/app/data/test.mp3' : 'data/test.mp3';
const { text } = await aiTranscriptions({
- model: model.model,
+ model,
fileStream: fs.createReadStream(path),
headers
});
diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
index caba7d922..dfa34727f 100644
--- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
+++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
@@ -43,9 +43,12 @@ export type PostPreviewFilesChunksProps = {
externalFileId?: string;
};
export type PreviewChunksResponse = {
- q: string;
- a: string;
-}[];
+ chunks: {
+ q: string;
+ a: string;
+ }[];
+ total: number;
+};
async function handler(
req: ApiRequestProps
@@ -123,13 +126,17 @@ async function handler(
customPdfParse
});
- return rawText2Chunks({
+ const chunks = rawText2Chunks({
rawText,
chunkSize,
maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
overlapRatio,
customReg: chunkSplitter ? [chunkSplitter] : [],
isQAImport: isQAImport
- }).slice(0, 10);
+ });
+ return {
+ chunks: chunks.slice(0, 10),
+ total: chunks.length
+ };
}
export default NextAPI(handler);
diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts
index 5f8581595..6a6d896ff 100644
--- a/projects/app/src/pages/api/v1/audio/transcriptions.ts
+++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts
@@ -66,7 +66,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
// }
const result = await aiTranscriptions({
- model: getDefaultSTTModel().model,
+ model: getDefaultSTTModel(),
fileStream: fs.createReadStream(file.path)
});
From 6ea57e4609a9c876d432a68b126497d6b5512684 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Mon, 24 Mar 2025 17:16:49 +0800
Subject: [PATCH 16/35] =?UTF-8?q?perf:=20ai=20proxy=20log=20remove=20retry?=
=?UTF-8?q?=20log=EF=BC=9Bperf:=20workflow=20type=20auto=20parse=EF=BC=9Ba?=
=?UTF-8?q?dd=20chunk=20spliter=20test=20(#4296)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* sync collection
* remove lock
* perf: workflow type auto parse
* add chunk spliter test
* perf: ai proxy log remove retry log
* udpate ai proxy field
---
.../zh-cn/docs/development/upgrading/492.md | 3 +
packages/global/core/workflow/constants.ts | 1 +
.../global/core/workflow/template/input.ts | 2 +-
.../service/core/workflow/dispatch/utils.ts | 10 +-
packages/web/i18n/en/account_model.json | 1 +
packages/web/i18n/zh-CN/account_model.json | 1 +
packages/web/i18n/zh-Hant/account_model.json | 1 +
projects/app/src/global/aiproxy/type.d.ts | 11 +-
.../account/model/Log/index.tsx | 31 +-
.../RenderInput/templates/Reference.tsx | 4 +-
projects/app/src/web/core/ai/channel.ts | 1 +
.../global/common/string/textSplitter.test.ts | 427 ++++++++++++++++++
12 files changed, 475 insertions(+), 18 deletions(-)
create mode 100644 test/cases/function/packages/global/common/string/textSplitter.test.ts
diff --git a/docSite/content/zh-cn/docs/development/upgrading/492.md b/docSite/content/zh-cn/docs/development/upgrading/492.md
index 96f8b5304..9146efbb9 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
@@ -26,6 +26,9 @@ weight: 799
3. 无 SSL 证书时复制失败,会提示弹窗用于手动复制。
4. FastGPT 未内置 ai proxy 渠道时,也能正常展示其名称。
5. 升级 nextjs 版本至 14.2.25。
+6. 工作流节点数组字符串类型,自动适配 string 输入。
+7. 工作流节点数组类型,自动进行 JSON parse 解析 string 输入。
+8. AI proxy 日志优化,去除重试失败的日志,仅保留最后一份错误日志。
## 🐛 修复
diff --git a/packages/global/core/workflow/constants.ts b/packages/global/core/workflow/constants.ts
index 5a0918883..d484f4f9f 100644
--- a/packages/global/core/workflow/constants.ts
+++ b/packages/global/core/workflow/constants.ts
@@ -20,6 +20,7 @@ export enum WorkflowIOValueTypeEnum {
number = 'number',
boolean = 'boolean',
object = 'object',
+
arrayString = 'arrayString',
arrayNumber = 'arrayNumber',
arrayBoolean = 'arrayBoolean',
diff --git a/packages/global/core/workflow/template/input.ts b/packages/global/core/workflow/template/input.ts
index c1e8ffe37..b4700c560 100644
--- a/packages/global/core/workflow/template/input.ts
+++ b/packages/global/core/workflow/template/input.ts
@@ -78,7 +78,7 @@ export const Input_Template_Text_Quote: FlowNodeInputItemType = {
export const Input_Template_File_Link: FlowNodeInputItemType = {
key: NodeInputKeyEnum.fileUrlList,
- renderTypeList: [FlowNodeInputTypeEnum.reference],
+ renderTypeList: [FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.input],
label: i18nT('app:workflow.user_file_input'),
debugLabel: i18nT('app:workflow.user_file_input'),
description: i18nT('app:workflow.user_file_input_desc'),
diff --git a/packages/service/core/workflow/dispatch/utils.ts b/packages/service/core/workflow/dispatch/utils.ts
index d73da3147..bcc7e8ae9 100644
--- a/packages/service/core/workflow/dispatch/utils.ts
+++ b/packages/service/core/workflow/dispatch/utils.ts
@@ -117,6 +117,9 @@ export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => {
return Boolean(value);
}
try {
+ if (WorkflowIOValueTypeEnum.arrayString && typeof value === 'string') {
+ return [value];
+ }
if (
type &&
[
@@ -124,7 +127,12 @@ export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => {
WorkflowIOValueTypeEnum.chatHistory,
WorkflowIOValueTypeEnum.datasetQuote,
WorkflowIOValueTypeEnum.selectApp,
- WorkflowIOValueTypeEnum.selectDataset
+ WorkflowIOValueTypeEnum.selectDataset,
+ WorkflowIOValueTypeEnum.arrayString,
+ WorkflowIOValueTypeEnum.arrayNumber,
+ WorkflowIOValueTypeEnum.arrayBoolean,
+ WorkflowIOValueTypeEnum.arrayObject,
+ WorkflowIOValueTypeEnum.arrayAny
].includes(type) &&
typeof value !== 'object'
) {
diff --git a/packages/web/i18n/en/account_model.json b/packages/web/i18n/en/account_model.json
index 9e0db08c2..7d974e6fd 100644
--- a/packages/web/i18n/en/account_model.json
+++ b/packages/web/i18n/en/account_model.json
@@ -37,6 +37,7 @@
"model_tokens": "Input/Output tokens",
"request_at": "Request time",
"request_duration": "Request duration: {{duration}}s",
+ "retry_times": "Number of retry times",
"running_test": "In testing",
"search_model": "Search for models",
"select_channel": "Select a channel name",
diff --git a/packages/web/i18n/zh-CN/account_model.json b/packages/web/i18n/zh-CN/account_model.json
index 852831120..66d450d2d 100644
--- a/packages/web/i18n/zh-CN/account_model.json
+++ b/packages/web/i18n/zh-CN/account_model.json
@@ -37,6 +37,7 @@
"model_tokens": "输入/输出 Tokens",
"request_at": "请求时间",
"request_duration": "请求时长: {{duration}}s",
+ "retry_times": "重试次数",
"running_test": "测试中",
"search_model": "搜索模型",
"select_channel": "选择渠道名",
diff --git a/packages/web/i18n/zh-Hant/account_model.json b/packages/web/i18n/zh-Hant/account_model.json
index fcbaed253..f592b39fe 100644
--- a/packages/web/i18n/zh-Hant/account_model.json
+++ b/packages/web/i18n/zh-Hant/account_model.json
@@ -35,6 +35,7 @@
"model_tokens": "輸入/輸出 Tokens",
"request_at": "請求時間",
"request_duration": "請求時長: {{duration}}s",
+ "retry_times": "重試次數",
"running_test": "測試中",
"search_model": "搜索模型",
"select_channel": "選擇渠道名",
diff --git a/projects/app/src/global/aiproxy/type.d.ts b/projects/app/src/global/aiproxy/type.d.ts
index 8238f2093..c8c63e66b 100644
--- a/projects/app/src/global/aiproxy/type.d.ts
+++ b/projects/app/src/global/aiproxy/type.d.ts
@@ -30,6 +30,13 @@ export type CreateChannelProps = {
};
// Log
+export type ChannelLogUsageType = {
+ cache_creation_tokens?: number;
+ cached_tokens?: number;
+ input_tokens?: number;
+ output_tokens?: number;
+ total_tokens?: number;
+};
export type ChannelLogListItemType = {
token_name: string;
model: string;
@@ -40,8 +47,8 @@ export type ChannelLogListItemType = {
created_at: number;
request_at: number;
code: number;
- prompt_tokens: number;
- completion_tokens: number;
+ usage?: ChannelLogUsageType;
endpoint: string;
content?: string;
+ retry_times?: number;
};
diff --git a/projects/app/src/pageComponents/account/model/Log/index.tsx b/projects/app/src/pageComponents/account/model/Log/index.tsx
index f57ad9d69..26e3dd39a 100644
--- a/projects/app/src/pageComponents/account/model/Log/index.tsx
+++ b/projects/app/src/pageComponents/account/model/Log/index.tsx
@@ -33,6 +33,7 @@ import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
import MyModal from '@fastgpt/web/components/common/MyModal';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
+import { ChannelLogUsageType } from '@/global/aiproxy/type';
type LogDetailType = {
id: number;
@@ -42,10 +43,10 @@ type LogDetailType = {
duration: number;
request_at: string;
code: number;
- prompt_tokens: number;
- completion_tokens: number;
+ usage?: ChannelLogUsageType;
endpoint: string;
+ retry_times?: number;
content?: string;
request_body?: string;
response_body?: string;
@@ -159,8 +160,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
duration: durationSecond,
request_at: formatTime2YMDHMS(item.request_at),
code: item.code,
- prompt_tokens: item.prompt_tokens,
- completion_tokens: item.completion_tokens,
+ usage: item.usage,
request_id: item.request_id,
endpoint: item.endpoint,
content: item.content
@@ -260,7 +260,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
{item.channelName} |
{item.model} |
- {item.prompt_tokens} / {item.completion_tokens}
+ {item.usage?.input_tokens} / {item.usage?.output_tokens}
|
10 ? 'red.600' : ''}>{item.duration.toFixed(2)}s |
@@ -297,6 +297,7 @@ const LogDetail = ({ data, onClose }: { data: LogDetailType; onClose: () => void
const { t } = useTranslation();
const { data: detailData } = useRequest2(
async () => {
+ console.log(data);
if (data.code === 200) return data;
try {
const res = await getLogDetail(data.id);
@@ -363,7 +364,7 @@ const LogDetail = ({ data, onClose }: { data: LogDetailType; onClose: () => void
RequestID
{detailData?.request_id}
-
+
{t('account_model:channel_status')}
{detailData?.code}
@@ -373,7 +374,7 @@ const LogDetail = ({ data, onClose }: { data: LogDetailType; onClose: () => void
Endpoint
{detailData?.endpoint}
-
+
{t('account_model:channel_name')}
{detailData?.channelName}
@@ -381,7 +382,7 @@ const LogDetail = ({ data, onClose }: { data: LogDetailType; onClose: () => void
{t('account_model:request_at')}
{detailData?.request_at}
-
+
{t('account_model:duration')}
{detailData?.duration.toFixed(2)}s
@@ -389,20 +390,26 @@ const LogDetail = ({ data, onClose }: { data: LogDetailType; onClose: () => void
{t('account_model:model')}
{detailData?.model}
-
+
{t('account_model:model_tokens')}
- {detailData?.prompt_tokens} / {detailData?.completion_tokens}
+ {detailData?.usage?.input_tokens} / {detailData?.usage?.output_tokens}
+ {detailData?.retry_times !== undefined && (
+
+ {t('account_model:retry_times')}
+ {detailData?.retry_times}
+
+ )}
{detailData?.content && (
-
+
Content
{detailData?.content}
)}
{detailData?.request_body && (
-
+
Request Body
{detailData?.request_body}
diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx
index 8c991ee37..8633b46d2 100644
--- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx
+++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx
@@ -247,9 +247,9 @@ const MultipleReferenceSelector = ({
// Get valid item and remove invalid item
const formatList = useMemo(() => {
- if (!value) return [];
+ if (!value || !Array.isArray(value)) return [];
- return value?.map((item) => {
+ return value.map((item) => {
const [nodeName, outputName] = getSelectValue(item);
return {
rawValue: item,
diff --git a/projects/app/src/web/core/ai/channel.ts b/projects/app/src/web/core/ai/channel.ts
index fd4534a3f..abd170e45 100644
--- a/projects/app/src/web/core/ai/channel.ts
+++ b/projects/app/src/web/core/ai/channel.ts
@@ -166,6 +166,7 @@ export const getChannelLog = (params: {
logs: ChannelLogListItemType[];
total: number;
}>(`/logs/search`, {
+ result_only: true,
request_id: params.request_id,
channel: params.channel,
model_name: params.model_name,
diff --git a/test/cases/function/packages/global/common/string/textSplitter.test.ts b/test/cases/function/packages/global/common/string/textSplitter.test.ts
new file mode 100644
index 000000000..0407cf4da
--- /dev/null
+++ b/test/cases/function/packages/global/common/string/textSplitter.test.ts
@@ -0,0 +1,427 @@
+import { it, expect } from 'vitest'; // 必须显式导入
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
+
+// 简单的嵌套测试
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `# A
+
+af da da fda a a
+
+## B
+
+阿凡撒发生的都是发大水
+
+### c
+
+dsgsgfsgs22
+
+#### D
+
+dsgsgfsgs22
+
+##### E
+
+dsgsgfsgs22sddddddd
+`,
+ result: [
+ `# A
+
+af da da fda a a`,
+ `# A
+## B
+
+阿凡撒发生的都是发大水`,
+ `# A
+## B
+### c
+
+dsgsgfsgs22`,
+ `# A
+## B
+### c
+#### D
+
+dsgsgfsgs22`,
+ `# A
+## B
+### c
+#### D
+##### E
+
+dsgsgfsgs22sddddddd`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 2000 });
+ expect(chunks).toEqual(mock.result);
+});
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `# A
+
+af da da fda a a
+
+### D
+
+dsgsgfsgs22`,
+ result: [
+ `# A
+
+af da da fda a a`,
+ `# A
+### D
+
+dsgsgfsgs22`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 2000 });
+ expect(chunks).toEqual(mock.result);
+});
+
+// 普通文本测试:单段不超过 500 字符
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `快速了解 FastGPT
+FastGPT 的能力与优势
+
+FastGPT 是一个基于 LLM 大语言模型的知识库问答系统,提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的问答场景!
+
+FastGPT 在线使用:https://tryfastgpt.ai
+
+FastGPT 能力
+1. 专属 AI 客服
+通过导入文档或已有问答对进行训练,让 AI 模型能根据你的文档以交互式对话方式回答问题。
+
+2. 简单易用的可视化界面
+FastGPT 采用直观的可视化界面设计,为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤,可以轻松完成 AI 客服的创建和训练流程。
+
+3. 自动数据预处理
+提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。
+
+4. 工作流编排
+基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。
+
+5. 强大的 API 集成
+FastGPT 对外的 API 接口对齐了 OpenAI 官方接口,可以直接接入现有的 GPT 应用,也可以轻松集成到企业微信、公众号、飞书等平台。
+
+FastGPT 特点
+项目开源
+
+FastGPT 遵循附加条件 Apache License 2.0 开源协议,你可以 Fork 之后进行二次开发和发布。FastGPT 社区版将保留核心功能,商业版仅在社区版基础上使用 API 的形式进行扩展,不影响学习使用。
+
+独特的 QA 结构
+
+针对客服问答场景设计的 QA 结构,提高在大量数据场景中的问答准确性。
+
+可视化工作流
+
+通过 Flow 模块展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。
+
+无限扩展
+
+基于 API 进行扩展,无需修改 FastGPT 源码,也可快速接入现有的程序中。
+
+便于调试
+
+提供搜索测试、引用修改、完整对话预览等多种调试途径。
+
+支持多种模型
+
+支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
+
+知识库核心流程
+
+FastGPT AI 相关参数配置说明
+
+在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。
+
+返回AI内容(高级编排特有)
+这是一个开关,打开的时候,当 AI 对话模块运行时,会将其输出的内容返回到浏览器(API响应);如果关闭,AI 输出的内容不会返回到浏览器,但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。
+
+最大上下文
+代表模型最多容纳的文字数量。
+
+函数调用
+支持函数调用的模型,在使用工具时更加准确。
+
+温度
+越低回答越严谨,少废话(实测下来,感觉差别不大)
+
+回复上限
+最大回复 token 数量。注意,是回复的Tokens!不是上下文 tokens。
+
+系统提示词
+被放置在上下文数组的最前面,role 为 system,用于引导模型。
+
+引用模板 & 引用提示词
+这两个参数与知识库问答场景相关,可以控制知识库相关的提示词。
+
+AI 对话消息组成
+想使用明白这两个变量,首先要了解传递传递给 AI 模型的消息格式。它是一个数组,FastGPT 中这个数组的组成形式为:
+
+[
+内置提示词(config.json 配置,一般为空)
+系统提示词 (用户输入的提示词)
+历史记录
+问题(由引用提示词、引用模板和用户问题组成)
+]
+🍅
+
+Tips: 可以通过点击上下文按键查看完整的上下文组成,便于调试。
+
+引用模板和提示词设计
+简易模式已移除该功能,仅在工作流中可配置,可点击工作流中AI对话节点内,知识库引用旁边的setting icon进行配置。随着模型的增强,这部分功能将逐步弱化。
+
+引用模板和引用提示词通常是成对出现,引用提示词依赖引用模板。
+
+FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变量)的格式存储,在转义成字符串时候会根据引用模板来进行格式化。知识库包含多个可用变量: q, a, sourceId(数据的ID), index(第n个数据), source(数据的集合名、文件名),score(距离得分,0-1) 可以通过 {{q}} {{a}} {{sourceId}} {{index}} {{source}} {{score}} 按需引入。下面一个模板例子:
+
+可以通过 知识库结构讲解 了解详细的知识库的结构。
+
+引用模板
+{instruction:"{{q}}",output:"{{a}}",source:"{{source}}"}
+搜索到的知识库,会自动将 q,a,source 替换成对应的内容。每条搜索到的内容,会通过 \n 隔开。例如:
+
+{instruction:"电影《铃芽之旅》的导演是谁?",output:"电影《铃芽之旅》的导演是新海诚。",source:"手动输入"}
+{instruction:"本作的主人公是谁?",output:"本作的主人公是名叫铃芽的少女。",source:""}
+{instruction:"电影《铃芽之旅》男主角是谁?",output:"电影《铃芽之旅》男主角是宗像草太,由松村北斗配音。",source:""}
+{instruction:"电影《铃芽之旅》的编剧是谁?22",output:"新海诚是本片的编剧。",source:"手动输入"}
+引用提示词
+引用模板需要和引用提示词一起使用,提示词中可以写引用模板的格式说明以及对话的要求等。可以使用 {{quote}} 来使用 引用模板,使用 {{question}} 来引入问题。例如:
+
+你的背景知识:
+"""
+{{quote}}
+"""
+对话要求:
+1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
+2. 使用背景知识回答问题。
+3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
+我的问题是:"{{question}}"
+转义后则为:
+
+你的背景知识:
+"""
+{instruction:"电影《铃芽之旅》的导演是谁?",output:"电影《铃芽之旅》的导演是新海诚。",source:"手动输入"}
+{instruction:"本作的主人公是谁?",output:"本作的主人公是名叫铃芽的少女。",source:""}
+{instruction:"电影《铃芽之旅》男主角是谁?",output:"电影《铃芽之旅》男主角是宗像草太,由松村北斗配音}
+"""
+对话要求:
+1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
+2. 使用背景知识回答问题。
+3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
+我的问题是:"{{question}}"
+总结
+引用模板规定了搜索出来的内容如何组成一句话,其由 q,a,index,source 多个变量组成。
+
+引用提示词由引用模板和提示词组成,提示词通常是对引用模板的一个描述,加上对模型的要求。
+
+引用模板和提示词设计 示例
+通用模板与问答模板对比
+我们通过一组你是谁的手动数据,对通用模板与问答模板的效果进行对比。此处特意打了个搞笑的答案,通用模板下 GPT35 就变得不那么听话了,`,
+ result: [
+ `快速了解 FastGPT
+FastGPT 的能力与优势
+
+FastGPT 是一个基于 LLM 大语言模型的知识库问答系统,提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的问答场景!
+
+FastGPT 在线使用:https://tryfastgpt.ai
+
+FastGPT 能力
+1. 专属 AI 客服
+通过导入文档或已有问答对进行训练,让 AI 模型能根据你的文档以交互式对话方式回答问题。
+2. 简单易用的可视化界面
+FastGPT 采用直观的可视化界面设计,为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤,可以轻松完成 AI 客服的创建和训练流程。
+3. 自动数据预处理
+提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。`,
+ `4. 工作流编排
+基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。
+5. 强大的 API 集成
+FastGPT 对外的 API 接口对齐了 OpenAI 官方接口,可以直接接入现有的 GPT 应用,也可以轻松集成到企业微信、公众号、飞书等平台。
+
+FastGPT 特点
+项目开源
+
+FastGPT 遵循附加条件 Apache License 2.0 开源协议,你可以 Fork 之后进行二次开发和发布。FastGPT 社区版将保留核心功能,商业版仅在社区版基础上使用 API 的形式进行扩展,不影响学习使用。
+
+独特的 QA 结构
+
+针对客服问答场景设计的 QA 结构,提高在大量数据场景中的问答准确性。
+
+可视化工作流
+
+通过 Flow 模块展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。
+
+无限扩展
+
+基于 API 进行扩展,无需修改 FastGPT 源码,也可快速接入现有的程序中。
+
+便于调试
+
+提供搜索测试、引用修改、完整对话预览等多种调试途径。
+
+支持多种模型`,
+ `支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
+
+知识库核心流程
+
+FastGPT AI 相关参数配置说明
+
+在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。
+
+返回AI内容(高级编排特有)
+这是一个开关,打开的时候,当 AI 对话模块运行时,会将其输出的内容返回到浏览器(API响应);如果关闭,AI 输出的内容不会返回到浏览器,但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。
+
+最大上下文
+代表模型最多容纳的文字数量。
+
+函数调用
+支持函数调用的模型,在使用工具时更加准确。
+
+温度
+越低回答越严谨,少废话(实测下来,感觉差别不大)
+
+回复上限
+最大回复 token 数量。注意,是回复的Tokens!不是上下文 tokens。
+
+系统提示词
+被放置在上下文数组的最前面,role 为 system,用于引导模型。
+
+引用模板 & 引用提示词
+这两个参数与知识库问答场景相关,可以控制知识库相关的提示词。`,
+ `AI 对话消息组成
+想使用明白这两个变量,首先要了解传递传递给 AI 模型的消息格式。它是一个数组,FastGPT 中这个数组的组成形式为:
+[
+内置提示词(config.json 配置,一般为空)
+系统提示词 (用户输入的提示词)
+历史记录
+问题(由引用提示词、引用模板和用户问题组成)
+]
+🍅
+Tips: 可以通过点击上下文按键查看完整的上下文组成,便于调试。
+引用模板和提示词设计
+简易模式已移除该功能,仅在工作流中可配置,可点击工作流中AI对话节点内,知识库引用旁边的setting icon进行配置。随着模型的增强,这部分功能将逐步弱化。
+引用模板和引用提示词通常是成对出现,引用提示词依赖引用模板。
+FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变量)的格式存储,在转义成字符串时候会根据引用模板来进行格式化。`,
+ `引用模板和引用提示词通常是成对出现,引用提示词依赖引用模板。
+FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变量)的格式存储,在转义成字符串时候会根据引用模板来进行格式化。知识库包含多个可用变量: q, a, sourceId(数据的ID), index(第n个数据), source(数据的集合名、文件名),score(距离得分,0-1) 可以通过 {{q}} {{a}} {{sourceId}} {{index}} {{source}} {{score}} 按需引入。下面一个模板例子:
+可以通过 知识库结构讲解 了解详细的知识库的结构。
+
+引用模板
+{instruction:"{{q}}",output:"{{a}}",source:"{{source}}"}
+搜索到的知识库,会自动将 q,a,source 替换成对应的内容。每条搜索到的内容,会通过
+ 隔开。例如:`,
+ `{instruction:"电影《铃芽之旅》的导演是谁?",output:"电影《铃芽之旅》的导演是新海诚。",source:"手动输入"}
+{instruction:"本作的主人公是谁?",output:"本作的主人公是名叫铃芽的少女。",source:""}
+{instruction:"电影《铃芽之旅》男主角是谁?",output:"电影《铃芽之旅》男主角是宗像草太,由松村北斗配音。",source:""}
+{instruction:"电影《铃芽之旅》的编剧是谁?22",output:"新海诚是本片的编剧。",source:"手动输入"}
+引用提示词
+引用模板需要和引用提示词一起使用,提示词中可以写引用模板的格式说明以及对话的要求等。可以使用 {{quote}} 来使用 引用模板,使用 {{question}} 来引入问题。例如:
+
+你的背景知识:
+"""
+{{quote}}
+"""
+对话要求:`,
+ `1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
+2. 使用背景知识回答问题。
+3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
+我的问题是:"{{question}}"
+转义后则为:
+
+你的背景知识:
+"""
+{instruction:"电影《铃芽之旅》的导演是谁?",output:"电影《铃芽之旅》的导演是新海诚。",source:"手动输入"}
+{instruction:"本作的主人公是谁?",output:"本作的主人公是名叫铃芽的少女。",source:""}
+{instruction:"电影《铃芽之旅》男主角是谁?",output:"电影《铃芽之旅》男主角是宗像草太,由松村北斗配音}
+"""
+对话要求:
+1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
+2. 使用背景知识回答问题。`,
+ `3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
+我的问题是:"{{question}}"
+总结
+引用模板规定了搜索出来的内容如何组成一句话,其由 q,a,index,source 多个变量组成。
+
+引用提示词由引用模板和提示词组成,提示词通常是对引用模板的一个描述,加上对模型的要求。
+
+引用模板和提示词设计 示例
+通用模板与问答模板对比
+我们通过一组你是谁的手动数据,对通用模板与问答模板的效果进行对比。此处特意打了个搞笑的答案,通用模板下 GPT35 就变得不那么听话了,`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 500 });
+
+ const normalizedChunks = chunks.map((chunk) => chunk.replace(/\n/g, ''));
+ const normalizedExpected = mock.result.map((result) => result.replace(/\n/g, ''));
+
+ expect(normalizedChunks).toEqual(normalizedExpected);
+});
+
+// 普通文本测试:单段超过 500 字符,有 20% 重叠
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`,
+ result: [
+ `FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。`,
+ `其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`,
+ `其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。`,
+ `可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`,
+ `其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。`,
+ `可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 500, overlapRatio: 0.2 });
+
+ expect(chunks).toEqual(mock.result);
+});
+
+// 自定义分隔符测试:分割后,内容少
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `这是测试文本 1
+----
+这是测试文本 2,
+---
+----
+这是测试文本 3
+----
+这是测试文本 4
+----`,
+ result: [
+ `这是测试文本 1`,
+ `这是测试文本 2,
+---`,
+ `这是测试文本 3`,
+ `这是测试文本 4`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 500, customReg: ['----'] });
+
+ expect(chunks).toEqual(mock.result);
+});
+
+// 自定义分隔符测试:分割后,内容超长,二次分割,有重叠。
+it(`Test splitText2Chunks`, () => {
+ const mock = {
+ text: `这是测试文本 1,短的
+----
+这是测试文本 2,长的。
+FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`,
+ result: [
+ `这是测试文本 1,短的`,
+ `这是测试文本 2,长的。
+FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。`,
+ `其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 500, customReg: ['----'] });
+
+ expect(chunks).toEqual(mock.result);
+});
From 5a47af6fffaceeb03c83e4a5fe78377d1f4cb1d9 Mon Sep 17 00:00:00 2001
From: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Date: Tue, 25 Mar 2025 00:10:26 +0800
Subject: [PATCH 17/35] pref: member/org/gourp list (#4295)
* refactor: org api
* refactor: org api
* pref: member/org/group list
* feat: change group owner api
* fix: manage org member
* pref: member search
---
.../support/permission/memberGroup/type.d.ts | 20 +-
.../global/support/user/team/org/type.d.ts | 2 +
packages/global/support/user/team/type.d.ts | 1 +
.../memberGroup/memberGroupSchema.ts | 1 +
.../support/permission/org/controllers.ts | 2 +-
.../components/common/Avatar/AvatarGroup.tsx | 17 +-
.../permission/MemberManager/MemberModal.tsx | 221 ++++++++------
.../support/user/team/OrgTags/index.tsx | 8 +-
.../team/GroupManage/GroupManageMember.tsx | 49 +--
.../GroupManage/GroupTransferOwnerModal.tsx | 42 +--
.../account/team/GroupManage/index.tsx | 39 +--
.../account/team/MemberTable.tsx | 22 +-
.../team/OrgManage/OrgMemberManageModal.tsx | 26 +-
.../account/team/OrgManage/index.tsx | 279 ++++++++++--------
.../account/team/PermissionManage/index.tsx | 1 +
.../pageComponents/account/team/context.tsx | 2 +-
projects/app/src/pages/account/team/index.tsx | 4 +-
projects/app/src/web/support/user/api.ts | 3 +-
.../src/web/support/user/team/group/api.ts | 11 +-
.../app/src/web/support/user/team/org/api.ts | 8 +-
.../app/src/web/support/user/useUserStore.ts | 19 --
21 files changed, 413 insertions(+), 364 deletions(-)
diff --git a/packages/global/support/permission/memberGroup/type.d.ts b/packages/global/support/permission/memberGroup/type.d.ts
index 7045cf534..f70e9a89e 100644
--- a/packages/global/support/permission/memberGroup/type.d.ts
+++ b/packages/global/support/permission/memberGroup/type.d.ts
@@ -19,9 +19,23 @@ type GroupMemberSchemaType = {
type MemberGroupType = MemberGroupSchemaType & {
members: {
tmbId: string;
- role: `${GroupMemberRole}`;
- }[]; // we can get tmb's info from other api. there is no need but only need to get tmb's id
- permission: TeamPermission;
+ name: string;
+ avatar: string;
+ }[];
+ count: number;
+ owner: {
+ tmbId: string;
+ name: string;
+ avatar: string;
+ };
+ canEdit: boolean;
};
type MemberGroupListType = MemberGroupType[];
+
+type GroupMemberItemType = {
+ tmbId: string;
+ name: string;
+ avatar: string;
+ role: `${GroupMemberRole}`;
+};
diff --git a/packages/global/support/user/team/org/type.d.ts b/packages/global/support/user/team/org/type.d.ts
index b742ece48..b25384a6e 100644
--- a/packages/global/support/user/team/org/type.d.ts
+++ b/packages/global/support/user/team/org/type.d.ts
@@ -1,5 +1,6 @@
import type { TeamPermission } from 'support/permission/user/controller';
import { ResourcePermissionType } from '../type';
+import { SourceMemberType } from 'support/user/type';
type OrgSchemaType = {
_id: string;
@@ -23,4 +24,5 @@ type OrgType = Omit & {
avatar: string;
permission: TeamPermission;
members: OrgMemberSchemaType[];
+ total: number; // members + children orgs
};
diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts
index 3be913870..c715d93b5 100644
--- a/packages/global/support/user/team/type.d.ts
+++ b/packages/global/support/user/team/type.d.ts
@@ -82,6 +82,7 @@ export type TeamMemberItemType = {
contact?: string;
createTime: Date;
updateTime?: Date;
+ orgs?: string[]; // full path name, pattern: /teamName/orgname1/orgname2
};
export type TeamTagItemType = {
diff --git a/packages/service/support/permission/memberGroup/memberGroupSchema.ts b/packages/service/support/permission/memberGroup/memberGroupSchema.ts
index 6964785bc..cc4ee05bc 100644
--- a/packages/service/support/permission/memberGroup/memberGroupSchema.ts
+++ b/packages/service/support/permission/memberGroup/memberGroupSchema.ts
@@ -1,6 +1,7 @@
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, getMongoModel } from '../../../common/mongo';
import { MemberGroupSchemaType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { GroupMemberCollectionName } from './groupMemberSchema';
const { Schema } = connectionMongo;
export const MemberGroupCollectionName = 'team_member_groups';
diff --git a/packages/service/support/permission/org/controllers.ts b/packages/service/support/permission/org/controllers.ts
index ce0746055..dd0d7dbab 100644
--- a/packages/service/support/permission/org/controllers.ts
+++ b/packages/service/support/permission/org/controllers.ts
@@ -90,6 +90,6 @@ export async function createRootOrg({
path: ''
}
],
- { session }
+ { session, ordered: true }
);
}
diff --git a/packages/web/components/common/Avatar/AvatarGroup.tsx b/packages/web/components/common/Avatar/AvatarGroup.tsx
index 6f9f3ce03..947e488bf 100644
--- a/packages/web/components/common/Avatar/AvatarGroup.tsx
+++ b/packages/web/components/common/Avatar/AvatarGroup.tsx
@@ -10,7 +10,16 @@ import { Box, Flex } from '@chakra-ui/react';
* @param [groupId] - group id to make the key unique
* @returns
*/
-function AvatarGroup({ avatars, max = 3 }: { max?: number; avatars: string[] }) {
+function AvatarGroup({
+ avatars,
+ max = 3,
+ total
+}: {
+ max?: number;
+ avatars: string[];
+ total?: number;
+}) {
+ const remain = total ?? avatars.length - max;
return (
{avatars.slice(0, max).map((avatar, index) => (
@@ -24,10 +33,10 @@ function AvatarGroup({ avatars, max = 3 }: { max?: number; avatars: string[] })
borderRadius={'50%'}
/>
))}
- {avatars.length > max && (
+ {remain > 0 && (
- +{avatars.length - max}
+ +{String(remain)}
)}
diff --git a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
index ab6138c35..da5468d8f 100644
--- a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
+++ b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
@@ -35,7 +35,7 @@ import { useContextSelector } from 'use-context-selector';
import { CollaboratorContext } from './context';
import { getTeamMembers } from '@/web/support/user/team/api';
import { getGroupList } from '@/web/support/user/team/group/api';
-import { getOrgList } from '@/web/support/user/team/org/api';
+import { getOrgList, getOrgMembers } from '@/web/support/user/team/org/api';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import MemberItemCard from './MemberItemCard';
import { GetSearchUserGroupOrg } from '@/web/support/user/api';
@@ -57,14 +57,52 @@ function MemberModal({
const collaboratorList = useContextSelector(CollaboratorContext, (v) => v.collaboratorList);
const [searchText, setSearchText] = useState('');
const [filterClass, setFilterClass] = useState<'member' | 'org' | 'group'>();
- const { data: members, ScrollData } = useScrollPagination(getTeamMembers, {
+ const [path, setPath] = useState('');
+ const [orgStack, setOrgStack] = useState([]);
+ const currentOrg = useMemo(() => orgStack[orgStack.length - 1], [orgStack]);
+
+ const { data: members, ScrollData: TeamMemberScrollData } = useScrollPagination(getTeamMembers, {
pageSize: 15
});
- const { data: [groups = [], orgs = []] = [], loading: loadingGroupsAndOrgs } = useRequest2(
+ const [rootOrg, setRootOrg] = useState();
+ const { data: orgMembers = [], ScrollData: OrgMemberScrollData } = useScrollPagination(
+ getOrgMembers,
+ {
+ pageSize: 20,
+ params: {
+ orgId: currentOrg?._id ?? rootOrg?._id
+ },
+ refreshDeps: [currentOrg?._id]
+ }
+ );
+ const onClickOrg = (org: OrgType) => {
+ setOrgStack([...orgStack, org]);
+ setPath(getOrgChildrenPath(org));
+ };
+
+ const { data: orgs = [] } = useRequest2(
+ () => {
+ const splitPath = path.split('/').filter(Boolean);
+ const orgs = orgStack.filter((o) => splitPath.includes(o.pathId));
+ setOrgStack(orgs);
+ return getOrgList(path);
+ },
+ {
+ manual: false,
+ refreshDeps: [path],
+ onSuccess: (data) => {
+ if (!rootOrg) {
+ setRootOrg(data[0]);
+ }
+ }
+ }
+ );
+
+ const { data: groups = [], loading: loadingGroupsAndOrgs } = useRequest2(
async () => {
- if (!userInfo?.team?.teamId) return [[], []];
- return Promise.all([getGroupList(), getOrgList()]);
+ if (!userInfo?.team?.teamId) return [];
+ return getGroupList();
},
{
manual: false,
@@ -72,69 +110,49 @@ function MemberModal({
}
);
- const [parentPath, setParentPath] = useState('');
-
const { data: searchedData } = useRequest2(() => GetSearchUserGroupOrg(searchText), {
manual: false,
throttleWait: 500,
+ debounceWait: 200,
refreshDeps: [searchText]
});
const paths = useMemo(() => {
- const splitPath = parentPath.split('/').filter(Boolean);
- return splitPath
- .map((id) => {
- const org = orgs.find((org) => org.pathId === id)!;
-
- if (org.path === '') return;
-
+ return orgStack
+ .map((org) => {
+ if (org?.path === '') return;
return {
parentId: getOrgChildrenPath(org),
parentName: org.name
};
})
.filter(Boolean) as ParentTreePathItemType[];
- }, [parentPath, orgs]);
+ }, [orgStack]);
const [selectedOrgIdList, setSelectedOrgIdList] = useState([]);
- const currentOrg = useMemo(() => {
- const splitPath = parentPath.split('/');
- const currentOrgId = splitPath[splitPath.length - 1];
- if (!currentOrgId) return;
- return orgs.find((org) => org.pathId === currentOrgId);
- }, [orgs, parentPath]);
const filterOrgs: (OrgType & { count?: number })[] = useMemo(() => {
if (searchText && searchedData) {
const orgids = searchedData.orgs.map((item) => item._id);
return orgs.filter((org) => orgids.includes(String(org._id)));
}
- if (!searchText && filterClass !== 'org') return [];
- if (parentPath === '') {
- setParentPath(`/${orgs[0].pathId}`);
- return [];
- }
return orgs
- .filter((org) => org.path === parentPath)
- .map((item) => ({
- ...item,
- count:
- item.members.length + orgs.filter((org) => org.path === getOrgChildrenPath(item)).length
+ .filter((org) => org.path !== '')
+ .map((org) => ({
+ ...org,
+ count: org.total
}));
- }, [searchText, filterClass, parentPath, orgs, searchedData]);
+ }, [searchText, orgs, searchedData]);
const [selectedMemberIdList, setSelectedMembers] = useState([]);
const filterMembers = useMemo(() => {
if (searchText) {
return searchedData?.members || [];
}
- if (!searchText && filterClass !== 'member' && filterClass !== 'org') return [];
- if (currentOrg && filterClass === 'org') {
- return members.filter((item) => currentOrg.members.find((v) => v.tmbId === item.tmbId));
- }
return members;
- }, [members, searchedData, searchText, filterClass, currentOrg]);
+ }, [searchText, members, searchedData?.members]);
+ console.log(filterMembers);
const [selectedGroupIdList, setSelectedGroupIdList] = useState([]);
const filterGroups = useMemo(() => {
@@ -197,19 +215,22 @@ function MemberModal({
id: `org-${item._id}`,
avatar: item.avatar,
name: item.name,
- onDelete: () => setSelectedOrgIdList(selectedOrgIdList.filter((v) => v !== item._id))
+ onDelete: () => setSelectedOrgIdList(selectedOrgIdList.filter((v) => v !== item._id)),
+ orgs: undefined
})),
...selectedGroups.map((item) => ({
id: `group-${item._id}`,
avatar: item.avatar,
name: item.name === DefaultGroupName ? userInfo?.team.teamName : item.name,
- onDelete: () => setSelectedGroupIdList(selectedGroupIdList.filter((v) => v !== item._id))
+ onDelete: () => setSelectedGroupIdList(selectedGroupIdList.filter((v) => v !== item._id)),
+ orgs: undefined
})),
...selectedMembers.map((item) => ({
id: `member-${item.tmbId}`,
avatar: item.avatar,
name: item.memberName,
- onDelete: () => setSelectedMembers(selectedMemberIdList.filter((v) => v !== item.tmbId))
+ onDelete: () => setSelectedMembers(selectedMemberIdList.filter((v) => v !== item.tmbId)),
+ orgs: item.orgs
}))
];
}, [
@@ -303,31 +324,57 @@ function MemberModal({
onClick={(parentId) => {
if (parentId === '') {
setFilterClass(undefined);
- setParentPath('');
+ setPath('');
} else if (
parentId === 'member' ||
parentId === 'org' ||
parentId === 'group'
) {
setFilterClass(parentId);
- setParentPath('');
+ setPath('');
} else {
- setParentPath(parentId);
+ setPath(parentId);
}
}}
rootName={t('common:common.Team')}
/>
)}
-
- {(filterClass === 'org' || filterClass === 'member') && (
- 0)) && (
+
- {filterOrgs?.map((org) => {
+ {filterMembers?.map((member) => {
+ const onChange = () => {
+ setSelectedMembers((state) => {
+ if (state.includes(member.tmbId)) {
+ return state.filter((v) => v !== member.tmbId);
+ }
+ return [...state, member.tmbId];
+ });
+ };
+ const collaborator = collaboratorList?.find((v) => v.tmbId === member.tmbId);
+ return (
+
+ );
+ })}
+
+ )}
+
+ {(filterClass === 'org' || searchText) &&
+ (() => {
+ const orgs = filterOrgs?.map((org) => {
const onChange = () => {
setSelectedOrgIdList((state) => {
if (state.includes(org._id)) {
@@ -374,47 +421,42 @@ function MemberModal({
bgColor: 'myGray.200'
}}
onClick={(e) => {
- setParentPath(getOrgChildrenPath(org));
+ onClickOrg(org);
+ // setPath(getOrgChildrenPath(org));
e.stopPropagation();
}}
/>
)}
);
- })}
- {filterMembers?.map((member) => {
- const onChange = () => {
- setSelectedMembers((state) => {
- if (state.includes(member.tmbId)) {
- return state.filter((v) => v !== member.tmbId);
- }
- return [...state, member.tmbId];
- });
- };
- const collaborator = collaboratorList?.find((v) => v.tmbId === member.tmbId);
- const memberOrgs = orgs.filter((org) =>
- org.members.find((v) => String(v.tmbId) === String(member.tmbId))
- );
- const memberPathIds = memberOrgs.map((org) =>
- (org.path + '/' + org.pathId).split('/').slice(0)
- );
- const memberOrgNames = memberPathIds.map((pathIds) =>
- pathIds.map((id) => orgs.find((v) => v.pathId === id)?.name).join('/')
- );
- return (
-
- );
- })}
-
- )}
+ });
+ return searchText ? (
+ orgs
+ ) : (
+
+ {orgs}
+ {orgMembers.map((member) => {
+ return (
+ {
+ setSelectedMembers((state) => {
+ if (state.includes(member.tmbId)) {
+ return state.filter((v) => v !== member.tmbId);
+ }
+ return [...state, member.tmbId];
+ });
+ }}
+ isChecked={selectedMemberIdList.includes(member.tmbId)}
+ orgs={member.orgs}
+ />
+ );
+ })}
+
+ );
+ })()}
{filterGroups?.map((group) => {
const onChange = () => {
setSelectedGroupIdList((state) => {
@@ -455,20 +497,7 @@ function MemberModal({
name={item.name ?? ''}
onChange={item.onDelete}
onDelete={item.onDelete}
- orgs={(() => {
- if (!item.id.startsWith('member-')) return [];
- const id = item.id.replace('member-', '');
- const memberOrgs = orgs.filter((org) =>
- org.members.find((v) => v.tmbId === id)
- );
- const memberPathIds = memberOrgs.map((org) =>
- (org.path + '/' + org.pathId).split('/').slice(0)
- );
- const memberOrgNames = memberPathIds.map((pathIds) =>
- pathIds.map((id) => orgs.find((v) => v.pathId === id)?.name).join('/')
- );
- return memberOrgNames;
- })()}
+ orgs={item?.orgs}
/>
);
})}
diff --git a/projects/app/src/components/support/user/team/OrgTags/index.tsx b/projects/app/src/components/support/user/team/OrgTags/index.tsx
index eebd6a26e..31f97a521 100644
--- a/projects/app/src/components/support/user/team/OrgTags/index.tsx
+++ b/projects/app/src/components/support/user/team/OrgTags/index.tsx
@@ -4,8 +4,8 @@ import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import Tag from '@fastgpt/web/components/common/Tag';
import React from 'react';
-function OrgTags({ orgs, type = 'simple' }: { orgs: string[]; type?: 'simple' | 'tag' }) {
- return (
+function OrgTags({ orgs, type = 'simple' }: { orgs?: string[]; type?: 'simple' | 'tag' }) {
+ return orgs?.length ? (
@@ -39,6 +39,10 @@ function OrgTags({ orgs, type = 'simple' }: { orgs: string[]; type?: 'simple' |
)}
+ ) : (
+
+ -
+
);
}
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index fbd8df13b..7cd017e2d 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -18,7 +18,7 @@ import React, { useMemo, useRef, useState } from 'react';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useContextSelector } from 'use-context-selector';
import { TeamContext } from '../context';
-import { putUpdateGroup } from '@/web/support/user/team/group/api';
+import { getGroupMembers, putUpdateGroup } from '@/web/support/user/team/group/api';
import { GroupMemberRole } from '@fastgpt/global/support/permission/memberGroup/constant';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
@@ -46,13 +46,26 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
return groups.find((item) => item._id === editGroupId);
}, [editGroupId, groups]);
+ const { data: groupMembers } = useRequest2(
+ () => {
+ if (editGroupId) return getGroupMembers(editGroupId);
+ return Promise.resolve(undefined);
+ },
+ {
+ manual: false,
+ onSuccess: (data) => {
+ setMembers(data ?? []);
+ }
+ }
+ );
+
const allMembers = useContextSelector(TeamContext, (v) => v.members);
const refetchMembers = useContextSelector(TeamContext, (v) => v.refetchMembers);
const MemberScrollData = useContextSelector(TeamContext, (v) => v.MemberScrollData);
const [hoveredMemberId, setHoveredMemberId] = useState();
const selectedMembersRef = useRef(null);
- const [members, setMembers] = useState(group?.members || []);
+ const [members, setMembers] = useState(groupMembers || []);
const [searchKey, setSearchKey] = useState('');
const filtered = useMemo(() => {
@@ -67,6 +80,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
const { runAsync: onUpdate, loading: isLoadingUpdate } = useRequest2(
async () => {
if (!editGroupId || !members.length) return;
+ console.log(members);
return putUpdateGroup({
groupId: editGroupId,
memberList: members
@@ -89,10 +103,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
}, [members, userInfo]);
const handleToggleSelect = (memberId: string) => {
- if (
- myRole === 'owner' &&
- memberId === group?.members.find((item) => item.role === 'owner')?.tmbId
- ) {
+ if (myRole === 'owner' && memberId === members.find((item) => item.role === 'owner')?.tmbId) {
toast({
title: t('user:team.group.toast.can_not_delete_owner'),
status: 'error'
@@ -102,7 +113,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
if (
myRole === 'admin' &&
- group?.members.find((item) => String(item.tmbId) === memberId)?.role !== 'member'
+ members.find((item) => String(item.tmbId) === memberId)?.role !== 'member'
) {
return;
}
@@ -110,7 +121,17 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
if (isSelected(memberId)) {
setMembers(members.filter((item) => item.tmbId !== memberId));
} else {
- setMembers([...members, { tmbId: memberId, role: 'member' }]);
+ const member = allMembers.find((m) => m.tmbId === memberId);
+ if (!member) return;
+ setMembers([
+ ...members,
+ {
+ name: member.memberName,
+ avatar: member.avatar,
+ tmbId: member.tmbId,
+ role: 'member'
+ }
+ ]);
}
};
@@ -188,7 +209,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
{t('common:chosen') + ': ' + members.length}
- {members.map((member) => {
+ {members?.map((member) => {
return (
setHoveredMemberId(member.tmbId)}
@@ -202,14 +223,8 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
_notLast={{ mb: 2 }}
>
- item.tmbId === member.tmbId)?.avatar}
- w="1.5rem"
- borderRadius={'md'}
- />
-
- {allMembers.find((item) => item.tmbId === member.tmbId)?.memberName}
-
+
+ {member.name}
{(() => {
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
index 3a583fbf0..aab5696b1 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
@@ -1,4 +1,4 @@
-import { putUpdateGroup } from '@/web/support/user/team/group/api';
+import { putGroupChangeOwner, putUpdateGroup } from '@/web/support/user/team/group/api';
import {
Box,
Flex,
@@ -38,10 +38,6 @@ export function ChangeOwnerModal({
return item.memberName.toLowerCase().includes(inputValue.toLowerCase());
});
- const OldOwnerId = useMemo(() => {
- return group?.members.find((item) => item.role === 'owner')?.tmbId;
- }, [group]);
-
const [keepAdmin, setKeepAdmin] = useState(true);
const {
@@ -52,36 +48,14 @@ export function ChangeOwnerModal({
const [selectedMember, setSelectedMember] = useState(null);
- const onChangeOwner = async (tmbId: string) => {
- if (!group) {
- return;
+ const { runAsync, loading } = useRequest2(
+ (tmbId: string) => putGroupChangeOwner(groupId, tmbId),
+ {
+ onSuccess: () => Promise.all([onClose(), refetchGroups()]),
+ successToast: t('common:permission.change_owner_success'),
+ errorToast: t('common:permission.change_owner_failed')
}
-
- const newMemberList = group.members
- .map((item) => {
- if (item.tmbId === OldOwnerId) {
- if (keepAdmin) {
- return { tmbId: OldOwnerId, role: 'admin' };
- }
- return { tmbId: OldOwnerId, role: 'member' };
- }
- return item;
- })
- .filter((item) => item.tmbId !== tmbId) as any;
-
- newMemberList.push({ tmbId, role: 'owner' });
-
- return putUpdateGroup({
- groupId,
- memberList: newMemberList
- });
- };
-
- const { runAsync, loading } = useRequest2(onChangeOwner, {
- onSuccess: () => Promise.all([onClose(), refetchGroups()]),
- successToast: t('common:permission.change_owner_success'),
- errorToast: t('common:permission.change_owner_failed')
- });
+ );
const onConfirm = async () => {
if (!selectedMember) {
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
index dc9ae458c..f411bc3f5 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
@@ -22,7 +22,7 @@ import MyMenu, { MenuItemType } from '@fastgpt/web/components/common/MyMenu';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { deleteGroup } from '@/web/support/user/team/group/api';
+import { deleteGroup, getGroupMembers } from '@/web/support/user/team/group/api';
import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant';
import MemberTag from '../../../../components/support/user/team/Info/MemberTag';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
@@ -39,10 +39,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { userInfo } = useUserStore();
- const { groups, refetchGroups, members, refetchMembers } = useContextSelector(
- TeamContext,
- (v) => v
- );
+ const { groups, refetchGroups, members, teamSize } = useContextSelector(TeamContext, (v) => v);
const [editGroup, setEditGroup] = useState();
const {
@@ -64,7 +61,6 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { runAsync: delDeleteGroup } = useRequest2(deleteGroup, {
onSuccess: () => {
refetchGroups();
- refetchMembers();
}
});
@@ -78,14 +74,9 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
onOpenManageGroupMember();
};
- const hasGroupManagePer = (group: (typeof groups)[0]) =>
- userInfo?.team.permission.hasManagePer ||
- ['admin', 'owner'].includes(
- group.members.find((item) => item.tmbId === userInfo?.team.tmbId)?.role ?? ''
- );
- const isGroupOwner = (group: (typeof groups)[0]) =>
- userInfo?.team.permission.hasManagePer ||
- group.members.find((item) => item.role === 'owner')?.tmbId === userInfo?.team.tmbId;
+ const hasGroupManagePer = (group: (typeof groups)[0]) => userInfo?.team.permission.hasManagePer;
+
+ const isGroupOwner = (group: (typeof groups)[0]) => userInfo?.team.permission.hasManagePer;
const {
isOpen: isOpenChangeOwner,
@@ -143,9 +134,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
}
avatar={group.avatar}
/>
-
- ({group.name === DefaultGroupName ? members.length : group.members.length})
-
+ ({group.name === DefaultGroupName ? teamSize : group.count})
|
@@ -153,26 +142,18 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
name={
group.name === DefaultGroupName
? members.find((item) => item.role === 'owner')?.memberName ?? ''
- : members.find(
- (item) =>
- item.tmbId ===
- group.members.find((item) => item.role === 'owner')?.tmbId
- )?.memberName ?? ''
+ : group.owner.name
}
avatar={
group.name === DefaultGroupName
? members.find((item) => item.role === 'owner')?.avatar ?? ''
- : members.find(
- (i) =>
- i.tmbId ===
- group.members.find((item) => item.role === 'owner')?.tmbId
- )?.avatar ?? ''
+ : group.owner.avatar
}
/>
|
{group.name === DefaultGroupName ? (
- v.avatar)} />
+ v.avatar)} total={teamSize} />
) : hasGroupManagePer(group) ? (
onManageMember(group)}>
@@ -180,6 +161,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
avatars={group.members.map(
(v) => members.find((m) => m.tmbId === v.tmbId)?.avatar ?? ''
)}
+ total={group.count}
/>
@@ -188,6 +170,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
avatars={group.members.map(
(v) => members.find((m) => m.tmbId === v.tmbId)?.avatar ?? ''
)}
+ total={group.count}
/>
)}
|
diff --git a/projects/app/src/pageComponents/account/team/MemberTable.tsx b/projects/app/src/pageComponents/account/team/MemberTable.tsx
index 7055244e6..d6232c549 100644
--- a/projects/app/src/pageComponents/account/team/MemberTable.tsx
+++ b/projects/app/src/pageComponents/account/team/MemberTable.tsx
@@ -30,8 +30,6 @@ import { TeamContext } from './context';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyIcon from '@fastgpt/web/components/common/Icon';
import dynamic from 'next/dynamic';
-import { useToast } from '@fastgpt/web/hooks/useToast';
-import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { delLeaveTeam } from '@/web/support/user/team/api';
import { GetSearchUserGroupOrg, postSyncMembers } from '@/web/support/user/api';
@@ -52,9 +50,8 @@ const TeamTagModal = dynamic(() => import('@/components/support/user/team/TeamTa
function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
- const { toast } = useToast();
- const { userInfo, teamPlanStatus } = useUserStore();
- const { feConfigs, setNotSufficientModalType } = useSystemStore();
+ const { userInfo } = useUserStore();
+ const { feConfigs } = useSystemStore();
const {
refetchGroups,
@@ -63,8 +60,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
members,
refetchMembers,
onSwitchTeam,
- MemberScrollData,
- orgs
+ MemberScrollData
} = useContextSelector(TeamContext, (v) => v);
const {
@@ -93,6 +89,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{
manual: false,
throttleWait: 500,
+ debounceWait: 200,
refreshDeps: [searchText]
}
);
@@ -281,16 +278,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{member.contact || '-'} |
{(() => {
- const memberOrgs = orgs.filter((org) =>
- org.members.find((v) => String(v.tmbId) === String(member.tmbId))
- );
- const memberPathIds = memberOrgs.map((org) =>
- (org.path + '/' + org.pathId).split('/').slice(0)
- );
- const memberOrgNames = memberPathIds.map((pathIds) =>
- pathIds.map((id) => orgs.find((v) => v.pathId === id)?.name).join('/')
- );
- return ;
+ return ;
})()}
|
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
index ca726383a..74e63bd8f 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
@@ -1,4 +1,4 @@
-import { putUpdateOrgMembers } from '@/web/support/user/team/org/api';
+import { getOrgMembers, putUpdateOrgMembers } from '@/web/support/user/team/org/api';
import {
Box,
Button,
@@ -18,10 +18,11 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import type React from 'react';
-import { useMemo, useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
import { useContextSelector } from 'use-context-selector';
import { TeamContext } from '../context';
import { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
export type GroupFormType = {
members: {
@@ -51,11 +52,21 @@ function OrgMemberManageModal({
}) {
const { t } = useTranslation();
const { members: allMembers, MemberScrollData } = useContextSelector(TeamContext, (v) => v);
+ const { data: orgMembers, ScrollData: OrgMemberScrollData } = useScrollPagination(getOrgMembers, {
+ pageSize: 20,
+ params: {
+ orgId: currentOrg?._id ?? ''
+ }
+ });
const [selectedMembers, setSelectedMembers] = useState(
- currentOrg.members.map((item) => item.tmbId)
+ orgMembers.map((item) => item.tmbId)
);
+ useEffect(() => {
+ setSelectedMembers(orgMembers.map((item) => item.tmbId));
+ }, [orgMembers]);
+
const [searchKey, setSearchKey] = useState('');
const filterMembers = useMemo(() => {
if (!searchKey) return allMembers;
@@ -150,9 +161,10 @@ function OrgMemberManageModal({
})}
-
- {`${t('common:chosen')}:${selectedMembers.length}`}
-
+ {/* */}
+
+
+ {`${t('common:chosen')}:${selectedMembers.length}`}
{selectedMembers.map((tmbId) => {
const member = allMembers.find((item) => item.tmbId === tmbId)!;
return (
@@ -179,7 +191,7 @@ function OrgMemberManageModal({
);
})}
-
+
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
index e4e2c5cda..5b5fc4f81 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
@@ -26,7 +26,12 @@ import { useMemo, useState } from 'react';
import { useContextSelector } from 'use-context-selector';
import MemberTag from '@/components/support/user/team/Info/MemberTag';
import { TeamContext } from '../context';
-import { deleteOrg, deleteOrgMember, getOrgList } from '@/web/support/user/team/org/api';
+import {
+ deleteOrg,
+ deleteOrgMember,
+ getOrgList,
+ getOrgMembers
+} from '@/web/support/user/team/org/api';
import IconButton from './IconButton';
import { defaultOrgForm, type OrgFormType } from './OrgInfoModal';
@@ -37,8 +42,9 @@ import Path from '@/components/common/folder/Path';
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
import { useSystemStore } from '@/web/common/system/useSystemStore';
-import { delRemoveMember } from '@/web/support/user/team/api';
+import { delRemoveMember, getTeamMembers } from '@/web/support/user/team/api';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
const OrgInfoModal = dynamic(() => import('./OrgInfoModal'));
const OrgMemberManageModal = dynamic(() => import('./OrgMemberManageModal'));
@@ -77,66 +83,63 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { userInfo, isTeamAdmin } = useUserStore();
const [searchOrg, setSearchOrg] = useState('');
+ const [orgStack, setOrgStack] = useState([]);
+ const currentOrg = useMemo(() => orgStack[orgStack.length - 1], [orgStack]);
+
+ const [rootOrg, setRootOrg] = useState();
+
+ const { data: members = [], ScrollData: MemberScrollData } = useScrollPagination(getOrgMembers, {
+ pageSize: 20,
+ params: {
+ orgId: currentOrg?._id ?? rootOrg?._id
+ },
+ refreshDeps: [currentOrg?._id, rootOrg?._id]
+ });
- const { members, MemberScrollData, refetchMembers } = useContextSelector(TeamContext, (v) => v);
const { feConfigs } = useSystemStore();
const isSyncMember = feConfigs.register_method?.includes('sync');
- const [parentPath, setParentPath] = useState('');
+ const [path, setPath] = useState('');
+
const {
data: orgs = [],
loading: isLoadingOrgs,
refresh: refetchOrgs
- } = useRequest2(getOrgList, {
- manual: false,
- refreshDeps: [userInfo?.team?.teamId]
- });
-
- const currentOrgs = useMemo(() => {
- if (orgs.length === 0) return [];
- if (parentPath === '') {
- const rootOrg = orgs.find((org) => org.path === '');
- if (rootOrg) {
- setParentPath(getOrgChildrenPath(rootOrg));
+ } = useRequest2(
+ () => {
+ // sync path to orgStack
+ const splitPath = path.split('/').filter(Boolean);
+ const orgs = orgStack.filter((o) => splitPath.includes(o.pathId));
+ setOrgStack(orgs);
+ return getOrgList(path);
+ },
+ {
+ manual: false,
+ refreshDeps: [userInfo?.team?.teamId, path],
+ onSuccess: (data) => {
+ if (!rootOrg) {
+ setRootOrg(data[0]);
+ }
}
- return [];
}
-
- return orgs
- .filter((org) => org.path === parentPath)
- .map((item) => {
- return {
- ...item,
- // Member + org
- count:
- item.members.length + orgs.filter((org) => org.path === getOrgChildrenPath(item)).length
- };
- });
- }, [orgs, parentPath]);
-
- const currentOrg = useMemo(() => {
- const splitPath = parentPath.split('/');
- const currentOrgId = splitPath[splitPath.length - 1];
- if (!currentOrgId) return;
-
- return orgs.find((org) => org.pathId === currentOrgId);
- }, [orgs, parentPath]);
+ );
const paths = useMemo(() => {
- const splitPath = parentPath.split('/').filter(Boolean);
- return splitPath
- .map((id) => {
- const org = orgs.find((org) => org.pathId === id)!;
-
+ return orgStack
+ .map((org) => {
if (org?.path === '') return;
-
return {
parentId: getOrgChildrenPath(org),
parentName: org.name
};
})
.filter(Boolean) as ParentTreePathItemType[];
- }, [parentPath, orgs]);
+ }, [orgStack]);
+
+ const onClickOrg = (org: OrgType) => {
+ setOrgStack([...orgStack, org]);
+ setPath(getOrgChildrenPath(org));
+ };
const [editOrg, setEditOrg] = useState();
const [manageMemberOrg, setManageMemberOrg] = useState();
@@ -174,7 +177,6 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
const { runAsync: deleteMemberFromTeamReq } = useRequest2(delRemoveMember, {
onSuccess: () => {
refetchOrgs();
- refetchMembers();
}
});
@@ -184,9 +186,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
return orgs
.filter((org) => org.name.includes(searchOrg))
.map((org) => ({
- ...org,
- count:
- org.members.length + orgs.filter((org) => org.path === getOrgChildrenPath(org)).length
+ ...org
}));
}, [orgs, searchOrg]);
@@ -210,11 +210,10 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
isLoading={isLoadingOrgs}
>
-
+
-
- {/* Table */}
+
@@ -229,21 +228,11 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
{searchedOrgs.map((org) => (
- setParentPath(getOrgChildrenPath(org))}
- >
+ onClickOrg(org)}>
|
- {
- setParentPath(getOrgChildrenPath(org));
- setSearchOrg('');
- }}
- >
+ onClickOrg(org)}>
- {org.count}
+ {org.total}
|
+ {isTeamAdmin && !isSyncMember && (
+
+ }
+ menuList={[
+ {
+ children: [
+ {
+ icon: 'edit',
+ label: t('account_team:edit_info'),
+ onClick: () => setEditOrg(org)
+ },
+ {
+ icon: 'common/file/move',
+ label: t('common:Move'),
+ onClick: () => setMovingOrg(org)
+ },
+ {
+ icon: 'delete',
+ label: t('account_team:delete'),
+ type: 'danger',
+ onClick: () => deleteOrgHandler(org._id)
+ }
+ ]
+ }
+ ]}
+ />
+ |
+ )}
))}
{!searchOrg &&
- currentOrgs.map((org) => (
-
- |
- {
- setParentPath(getOrgChildrenPath(org));
- setSearchOrg('');
- }}
- >
-
- {org.count}
-
-
- |
- {isTeamAdmin && !isSyncMember && (
-
- }
- menuList={[
- {
- children: [
- {
- icon: 'edit',
- label: t('account_team:edit_info'),
- onClick: () => setEditOrg(org)
- },
- {
- icon: 'common/file/move',
- label: t('common:Move'),
- onClick: () => setMovingOrg(org)
- },
- {
- icon: 'delete',
- label: t('account_team:delete'),
- type: 'danger',
- onClick: () => deleteOrgHandler(org._id)
- }
- ]
- }
- ]}
- />
+ orgs
+ .filter((org) => org.path !== '')
+ .map((org) => (
+ |
+ |
+ onClickOrg(org)}>
+
+ {org.total}
+
+
|
- )}
-
- ))}
+ {isTeamAdmin && !isSyncMember && (
+
+ }
+ menuList={[
+ {
+ children: [
+ {
+ icon: 'edit',
+ label: t('account_team:edit_info'),
+ onClick: () => setEditOrg(org)
+ },
+ {
+ icon: 'common/file/move',
+ label: t('common:Move'),
+ onClick: () => setMovingOrg(org)
+ },
+ {
+ icon: 'delete',
+ label: t('account_team:delete'),
+ type: 'danger',
+ onClick: () => deleteOrgHandler(org._id)
+ }
+ ]
+ }
+ ]}
+ />
+ |
+ )}
+
+ ))}
{!searchOrg &&
- currentOrg?.members.map((member) => {
- const memberInfo = members.find((m) => m.tmbId === member.tmbId);
- if (!memberInfo) return null;
-
+ members.map((member) => {
return (
|
-
+
|
{isTeamAdmin && (
@@ -333,14 +345,14 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
}
},
label: t('account_team:delete_from_team', {
- username: memberInfo.memberName
+ username: member.memberName
}),
onClick: () => {
openDeleteMemberFromTeamModal(
() => deleteMemberFromTeamReq(member.tmbId),
undefined,
t('account_team:confirm_delete_from_team', {
- username: memberInfo.memberName
+ username: member.memberName
})
)();
}
@@ -362,7 +374,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
deleteMemberReq(currentOrg._id, member.tmbId),
undefined,
t('account_team:confirm_delete_from_org', {
- username: memberInfo.memberName
+ username: member.memberName
})
)()
}
@@ -385,22 +397,29 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
{!isSyncMember && (
-
+
- {currentOrg?.name}
+ {currentOrg?.name || userInfo?.team.teamName}
- {currentOrg?.path !== '' && (
+ {currentOrg && currentOrg?.path !== '' && (
setEditOrg(currentOrg)} />
)}
- {currentOrg?.description || t('common:common.no_intro')}
+ {currentOrg && (
+ {currentOrg?.description || t('common:common.no_intro')}
+ )}
{t('common:common.Action')}
- {currentOrg && isTeamAdmin && (
+ {isTeamAdmin && (
{
setEditOrg({
...defaultOrgForm,
- parentId: currentOrg?._id
+ parentId: currentOrg?._id ?? rootOrg?._id
});
}}
/>
setManageMemberOrg(currentOrg)}
+ onClick={() => setManageMemberOrg(currentOrg ?? rootOrg)}
/>
- {currentOrg?.path !== '' && (
+ {currentOrg && currentOrg?.path !== '' && (
<>
GetSearchUserGroupOrg(searchKey), {
manual: false,
throttleWait: 500,
+ debounceWait: 200,
refreshDeps: [searchKey]
});
diff --git a/projects/app/src/pageComponents/account/team/context.tsx b/projects/app/src/pageComponents/account/team/context.tsx
index 509eda64f..b8cf4c88c 100644
--- a/projects/app/src/pageComponents/account/team/context.tsx
+++ b/projects/app/src/pageComponents/account/team/context.tsx
@@ -104,7 +104,7 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
refreshList: refetchMemberList,
ScrollData: MemberScrollData
} = useScrollPagination(getTeamMembers, {
- pageSize: 1000,
+ pageSize: 20,
params: {
withLeaved: true
}
diff --git a/projects/app/src/pages/account/team/index.tsx b/projects/app/src/pages/account/team/index.tsx
index a23c32c6f..5d613dbb5 100644
--- a/projects/app/src/pages/account/team/index.tsx
+++ b/projects/app/src/pages/account/team/index.tsx
@@ -48,7 +48,7 @@ const Team = () => {
const { t } = useTranslation();
const { userInfo } = useUserStore();
- const { setEditTeamData, isLoading, teamSize } = useContextSelector(TeamContext, (v) => v);
+ const { setEditTeamData, teamSize } = useContextSelector(TeamContext, (v) => v);
const Tabs = useMemo(
() => (
@@ -75,7 +75,7 @@ const Team = () => {
);
return (
-
+
{/* header */}
GET('/proApi/support/user/search', { searchKey, ...options });
+) =>
+ GET('/proApi/support/user/search', { searchKey, ...options }, { maxQuantity: 1 });
export const ExportMembers = () => GET<{ csv: string }>('/proApi/support/user/team/member/export');
diff --git a/projects/app/src/web/support/user/team/group/api.ts b/projects/app/src/web/support/user/team/group/api.ts
index e6d47f61c..7de009cfa 100644
--- a/projects/app/src/web/support/user/team/group/api.ts
+++ b/projects/app/src/web/support/user/team/group/api.ts
@@ -1,5 +1,8 @@
import { DELETE, GET, POST, PUT } from '@/web/common/api/request';
-import type { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
+import type {
+ GroupMemberItemType,
+ MemberGroupListType
+} from '@fastgpt/global/support/permission/memberGroup/type';
import type {
postCreateGroupData,
putUpdateGroupData
@@ -15,3 +18,9 @@ export const deleteGroup = (groupId: string) =>
export const putUpdateGroup = (data: putUpdateGroupData) =>
PUT('/proApi/support/user/team/group/update', data);
+
+export const getGroupMembers = (groupId: string) =>
+ GET(`/proApi/support/user/team/group/members`, { groupId });
+
+export const putGroupChangeOwner = (groupId: string, tmbId: string) =>
+ PUT(`/proApi/support/user/team/group/changeOwner`, { groupId, tmbId });
diff --git a/projects/app/src/web/support/user/team/org/api.ts b/projects/app/src/web/support/user/team/org/api.ts
index f25a48479..c33a5ced9 100644
--- a/projects/app/src/web/support/user/team/org/api.ts
+++ b/projects/app/src/web/support/user/team/org/api.ts
@@ -6,8 +6,11 @@ import type {
} from '@fastgpt/global/support/user/team/org/api';
import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
import type { putMoveOrgType } from '@fastgpt/global/support/user/team/org/api';
+import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
-export const getOrgList = () => GET('/proApi/support/user/team/org/list');
+export const getOrgList = (path: string) =>
+ GET(`/proApi/support/user/team/org/list`, { orgPath: path });
export const postCreateOrg = (data: postCreateOrgData) =>
POST('/proApi/support/user/team/org/create', data);
@@ -28,3 +31,6 @@ export const putUpdateOrgMembers = (data: putUpdateOrgMembersData) =>
// export const putChnageOrgOwner = (data: putChnageOrgOwnerData) =>
// PUT('/proApi/support/user/team/org/changeOwner', data);
+
+export const getOrgMembers = (data: PaginationProps<{ orgId: string }>) =>
+ GET>(`/proApi/support/user/team/org/members`, data);
diff --git a/projects/app/src/web/support/user/useUserStore.ts b/projects/app/src/web/support/user/useUserStore.ts
index 91128c703..a37715fac 100644
--- a/projects/app/src/web/support/user/useUserStore.ts
+++ b/projects/app/src/web/support/user/useUserStore.ts
@@ -32,8 +32,6 @@ type State = {
loadAndGetGroups: (init?: boolean) => Promise;
teamOrgs: OrgType[];
- myOrgs: OrgType[];
- loadAndGetOrgs: (init?: boolean) => Promise;
};
export const useUserStore = create()(
@@ -122,23 +120,6 @@ export const useUserStore = create()(
);
});
- return res;
- },
- myOrgs: [],
- loadAndGetOrgs: async (init = false) => {
- if (!useSystemStore.getState()?.feConfigs?.isPlus) return [];
-
- const randomRefresh = Math.random() > 0.7;
- if (!randomRefresh && !init && get().myOrgs.length) return Promise.resolve(get().myOrgs);
-
- const res = await getOrgList();
- set((state) => {
- state.teamOrgs = res;
- state.myOrgs = res.filter((item) =>
- item.members.map((i) => String(i.tmbId)).includes(String(state.userInfo?.team?.tmbId))
- );
- });
-
return res;
}
})),
From 826a53dcb65045a5eb68f31b284a8933421e54ea Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Tue, 25 Mar 2025 09:57:19 +0800
Subject: [PATCH 18/35] tmp org api rewrite (#4304)
* sync collection
* remove lock
* tmp org api rewrite
---
.../zh-cn/docs/development/upgrading/492.md | 1 +
.../global/support/user/team/org/type.d.ts | 13 +++-
.../memberGroup/memberGroupSchema.ts | 1 -
projects/app/package.json | 2 +-
.../team/GroupManage/GroupManageMember.tsx | 28 +++----
.../account/team/MemberTable.tsx | 2 +-
.../account/team/OrgManage/OrgInfoModal.tsx | 1 -
.../team/OrgManage/OrgMemberManageModal.tsx | 4 +-
.../account/team/OrgManage/OrgMoveModal.tsx | 8 +-
.../account/team/OrgManage/index.tsx | 76 ++++++++-----------
.../pageComponents/account/team/context.tsx | 21 +----
.../app/src/web/support/user/team/org/api.ts | 7 +-
.../app/src/web/support/user/useUserStore.ts | 1 -
13 files changed, 66 insertions(+), 99 deletions(-)
diff --git a/docSite/content/zh-cn/docs/development/upgrading/492.md b/docSite/content/zh-cn/docs/development/upgrading/492.md
index 9146efbb9..06999a3eb 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
@@ -18,6 +18,7 @@ weight: 799
2. 知识库分块增加自定义分隔符预设值,同时支持自定义换行符分割。
3. 外部变量改名:自定义变量。 并且支持在测试时调试,在分享链接中,该变量直接隐藏。
4. 集合同步时,支持同步修改标题。
+5. 团队成员管理重构,抽离主流 IM SSO(企微、飞书、钉钉),并支持通过自定义 SSO 接入 FastGPT。同时完善与外部系统的成员同步。
## ⚙️ 优化
diff --git a/packages/global/support/user/team/org/type.d.ts b/packages/global/support/user/team/org/type.d.ts
index b25384a6e..f2afac2d5 100644
--- a/packages/global/support/user/team/org/type.d.ts
+++ b/packages/global/support/user/team/org/type.d.ts
@@ -1,6 +1,6 @@
-import type { TeamPermission } from 'support/permission/user/controller';
+import type { TeamPermission } from '../../../permission/user/controller';
import { ResourcePermissionType } from '../type';
-import { SourceMemberType } from 'support/user/type';
+import { SourceMemberType } from '../../type';
type OrgSchemaType = {
_id: string;
@@ -8,7 +8,7 @@ type OrgSchemaType = {
pathId: string;
path: string;
name: string;
- avatar?: string;
+ avatar: string;
description?: string;
updateTime: Date;
};
@@ -20,7 +20,12 @@ type OrgMemberSchemaType = {
tmbId: string;
};
-type OrgType = Omit & {
+export type OrgListItemType = OrgSchemaType & {
+ permission: TeamPermission;
+ total: number; // members + children orgs
+};
+
+export type OrgType = Omit & {
avatar: string;
permission: TeamPermission;
members: OrgMemberSchemaType[];
diff --git a/packages/service/support/permission/memberGroup/memberGroupSchema.ts b/packages/service/support/permission/memberGroup/memberGroupSchema.ts
index cc4ee05bc..6964785bc 100644
--- a/packages/service/support/permission/memberGroup/memberGroupSchema.ts
+++ b/packages/service/support/permission/memberGroup/memberGroupSchema.ts
@@ -1,7 +1,6 @@
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, getMongoModel } from '../../../common/mongo';
import { MemberGroupSchemaType } from '@fastgpt/global/support/permission/memberGroup/type';
-import { GroupMemberCollectionName } from './groupMemberSchema';
const { Schema } = connectionMongo;
export const MemberGroupCollectionName = 'team_member_groups';
diff --git a/projects/app/package.json b/projects/app/package.json
index b37c1b5ad..2185f6f96 100644
--- a/projects/app/package.json
+++ b/projects/app/package.json
@@ -1,6 +1,6 @@
{
"name": "app",
- "version": "4.9.1",
+ "version": "4.9.2",
"private": false,
"scripts": {
"dev": "next dev",
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index 7cd017e2d..e0d1b05f4 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -24,6 +24,8 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
+import { GroupMemberItemType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { useMount } from 'ahooks';
export type GroupFormType = {
members: {
@@ -46,26 +48,20 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
return groups.find((item) => item._id === editGroupId);
}, [editGroupId, groups]);
- const { data: groupMembers } = useRequest2(
- () => {
- if (editGroupId) return getGroupMembers(editGroupId);
- return Promise.resolve(undefined);
- },
- {
- manual: false,
- onSuccess: (data) => {
- setMembers(data ?? []);
- }
- }
- );
-
const allMembers = useContextSelector(TeamContext, (v) => v.members);
const refetchMembers = useContextSelector(TeamContext, (v) => v.refetchMembers);
const MemberScrollData = useContextSelector(TeamContext, (v) => v.MemberScrollData);
const [hoveredMemberId, setHoveredMemberId] = useState();
const selectedMembersRef = useRef(null);
- const [members, setMembers] = useState(groupMembers || []);
+ const [members, setMembers] = useState([]);
+
+ useMount(async () => {
+ if (editGroupId) {
+ const data = await getGroupMembers(editGroupId);
+ setMembers(data);
+ }
+ });
const [searchKey, setSearchKey] = useState('');
const filtered = useMemo(() => {
@@ -80,7 +76,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
const { runAsync: onUpdate, loading: isLoadingUpdate } = useRequest2(
async () => {
if (!editGroupId || !members.length) return;
- console.log(members);
+
return putUpdateGroup({
groupId: editGroupId,
memberList: members
@@ -209,7 +205,7 @@ function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGro
{t('common:chosen') + ': ' + members.length}
- {members?.map((member) => {
+ {members.map((member) => {
return (
setHoveredMemberId(member.tmbId)}
diff --git a/projects/app/src/pageComponents/account/team/MemberTable.tsx b/projects/app/src/pageComponents/account/team/MemberTable.tsx
index d6232c549..994008bc6 100644
--- a/projects/app/src/pageComponents/account/team/MemberTable.tsx
+++ b/projects/app/src/pageComponents/account/team/MemberTable.tsx
@@ -268,7 +268,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{member.memberName}
{member.status !== 'active' && (
-
+
{t('account_team:leave')}
)}
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
index c2e49b368..c992fbead 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
@@ -7,7 +7,6 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import dynamic from 'next/dynamic';
import { useForm } from 'react-hook-form';
export type OrgFormType = {
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
index 74e63bd8f..13a3bb204 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
@@ -21,7 +21,7 @@ import type React from 'react';
import { useEffect, useMemo, useState } from 'react';
import { useContextSelector } from 'use-context-selector';
import { TeamContext } from '../context';
-import { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
export type GroupFormType = {
@@ -46,7 +46,7 @@ function OrgMemberManageModal({
refetchOrgs,
onClose
}: {
- currentOrg: OrgType;
+ currentOrg: OrgListItemType;
refetchOrgs: () => void;
onClose: () => void;
}) {
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
index 080c4eedb..96e2e705a 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
@@ -1,6 +1,6 @@
import { putMoveOrg } from '@/web/support/user/team/org/api';
import { Button, ModalBody, ModalFooter } from '@chakra-ui/react';
-import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import type { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
@@ -15,13 +15,13 @@ function OrgMoveModal({
onClose,
onSuccess
}: {
- movingOrg: OrgType;
- orgs: OrgType[];
+ movingOrg: OrgListItemType;
+ orgs: OrgListItemType[];
onClose: () => void;
onSuccess: () => void;
}) {
const { t } = useTranslation();
- const [selectedOrg, setSelectedOrg] = useState();
+ const [selectedOrg, setSelectedOrg] = useState();
const { userInfo } = useUserStore();
const team = userInfo?.team!;
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
index 5b5fc4f81..1c7fe5e53 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
@@ -14,7 +14,7 @@ import {
Tr,
VStack
} from '@chakra-ui/react';
-import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import type { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import Avatar from '@fastgpt/web/components/common/Avatar';
import MyIcon from '@fastgpt/web/components/common/Icon';
import type { IconNameType } from '@fastgpt/web/components/common/Icon/type';
@@ -23,9 +23,7 @@ import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import { useMemo, useState } from 'react';
-import { useContextSelector } from 'use-context-selector';
import MemberTag from '@/components/support/user/team/Info/MemberTag';
-import { TeamContext } from '../context';
import {
deleteOrg,
deleteOrgMember,
@@ -39,10 +37,10 @@ import { defaultOrgForm, type OrgFormType } from './OrgInfoModal';
import dynamic from 'next/dynamic';
import MyBox from '@fastgpt/web/components/common/MyBox';
import Path from '@/components/common/folder/Path';
-import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
import { useSystemStore } from '@/web/common/system/useSystemStore';
-import { delRemoveMember, getTeamMembers } from '@/web/support/user/team/api';
+import { delRemoveMember } from '@/web/support/user/team/api';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
@@ -82,24 +80,16 @@ function ActionButton({
function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { userInfo, isTeamAdmin } = useUserStore();
- const [searchOrg, setSearchOrg] = useState('');
- const [orgStack, setOrgStack] = useState([]);
- const currentOrg = useMemo(() => orgStack[orgStack.length - 1], [orgStack]);
-
- const [rootOrg, setRootOrg] = useState();
-
- const { data: members = [], ScrollData: MemberScrollData } = useScrollPagination(getOrgMembers, {
- pageSize: 20,
- params: {
- orgId: currentOrg?._id ?? rootOrg?._id
- },
- refreshDeps: [currentOrg?._id, rootOrg?._id]
- });
-
const { feConfigs } = useSystemStore();
-
const isSyncMember = feConfigs.register_method?.includes('sync');
- const [path, setPath] = useState('');
+
+ const [searchOrg, setSearchOrg] = useState('');
+
+ const [parentId, setParentId] = useState();
+ const [currentOrg, setCurrentOrg] = useState();
+
+ // 用于 org 层级
+ const [orgStack, setOrgStack] = useState([]);
const {
data: orgs = [],
@@ -107,43 +97,35 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
refresh: refetchOrgs
} = useRequest2(
() => {
- // sync path to orgStack
- const splitPath = path.split('/').filter(Boolean);
- const orgs = orgStack.filter((o) => splitPath.includes(o.pathId));
- setOrgStack(orgs);
- return getOrgList(path);
+ return getOrgList(parentId);
},
{
manual: false,
- refreshDeps: [userInfo?.team?.teamId, path],
- onSuccess: (data) => {
- if (!rootOrg) {
- setRootOrg(data[0]);
- }
- }
+ refreshDeps: [userInfo?.team?.teamId, parentId]
}
);
const paths = useMemo(() => {
+ if (!currentOrg) return [];
return orgStack
.map((org) => {
- if (org?.path === '') return;
return {
parentId: getOrgChildrenPath(org),
parentName: org.name
};
})
.filter(Boolean) as ParentTreePathItemType[];
- }, [orgStack]);
+ }, [currentOrg, orgStack]);
- const onClickOrg = (org: OrgType) => {
+ const onClickOrg = (org: OrgListItemType) => {
+ setParentId(currentOrg?._id);
setOrgStack([...orgStack, org]);
- setPath(getOrgChildrenPath(org));
+ setCurrentOrg(org);
};
const [editOrg, setEditOrg] = useState();
- const [manageMemberOrg, setManageMemberOrg] = useState();
- const [movingOrg, setMovingOrg] = useState();
+ const [manageMemberOrg, setManageMemberOrg] = useState();
+ const [movingOrg, setMovingOrg] = useState();
// Delete org
const { ConfirmModal: ConfirmDeleteOrgModal, openConfirm: openDeleteOrgModal } = useConfirm({
@@ -157,6 +139,14 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
}
});
+ const { data: members = [], ScrollData: MemberScrollData } = useScrollPagination(getOrgMembers, {
+ pageSize: 20,
+ params: {
+ orgId: currentOrg?._id
+ },
+ refreshDeps: [currentOrg?._id]
+ });
+
// Delete member
const { ConfirmModal: ConfirmDeleteMemberFromOrg, openConfirm: openDeleteMemberFromOrgModal } =
useConfirm({
@@ -183,11 +173,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
const searchedOrgs = useMemo(() => {
if (!searchOrg) return [];
- return orgs
- .filter((org) => org.name.includes(searchOrg))
- .map((org) => ({
- ...org
- }));
+ return orgs.filter((org) => org.name.includes(searchOrg));
}, [orgs, searchOrg]);
return (
@@ -231,7 +217,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
onClickOrg(org)}>
onClickOrg(org)}>
-
+
{org.total}
{
setEditOrg({
...defaultOrgForm,
- parentId: currentOrg?._id ?? rootOrg?._id
+ parentId: currentOrg?._id
});
}}
/>
diff --git a/projects/app/src/pageComponents/account/team/context.tsx b/projects/app/src/pageComponents/account/team/context.tsx
index b8cf4c88c..b1aa6730b 100644
--- a/projects/app/src/pageComponents/account/team/context.tsx
+++ b/projects/app/src/pageComponents/account/team/context.tsx
@@ -16,7 +16,6 @@ import { useTranslation } from 'next-i18next';
import { getGroupList } from '@/web/support/user/team/group/api';
import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
-import { getOrgList } from '@/web/support/user/team/org/api';
import { OrgType } from '@fastgpt/global/support/user/team/org/type';
const EditInfoModal = dynamic(() => import('./EditInfoModal'));
@@ -25,7 +24,6 @@ type TeamModalContextType = {
myTeams: TeamTmbItemType[];
members: TeamMemberItemType[];
groups: MemberGroupListType;
- orgs: OrgType[];
isLoading: boolean;
onSwitchTeam: (teamId: string) => void;
setEditTeamData: React.Dispatch>;
@@ -33,7 +31,6 @@ type TeamModalContextType = {
refetchMembers: () => void;
refetchTeams: () => void;
refetchGroups: () => void;
- refetchOrgs: () => void;
teamSize: number;
MemberScrollData: ReturnType['ScrollData'];
};
@@ -42,7 +39,6 @@ export const TeamContext = createContext({
myTeams: [],
groups: [],
members: [],
- orgs: [],
isLoading: false,
onSwitchTeam: function (_teamId: string): void {
throw new Error('Function not implemented.');
@@ -59,9 +55,6 @@ export const TeamContext = createContext({
refetchGroups: function (): void {
throw new Error('Function not implemented.');
},
- refetchOrgs: function (): void {
- throw new Error('Function not implemented.');
- },
teamSize: 0,
MemberScrollData: () => <>>
});
@@ -80,15 +73,6 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
refreshDeps: [userInfo?._id]
});
- const {
- data: orgs = [],
- loading: isLoadingOrgs,
- refresh: refetchOrgs
- } = useRequest2(getOrgList, {
- manual: false,
- refreshDeps: [userInfo?.team?.teamId]
- });
-
const { data: teamMemberCountData, refresh: refetchTeamMemberCount } = useRequest2(
getTeamMemberCount,
{
@@ -135,16 +119,13 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
refreshDeps: [userInfo?.team?.teamId]
});
- const isLoading =
- isLoadingTeams || isSwitchingTeam || loadingMembers || isLoadingGroups || isLoadingOrgs;
+ const isLoading = isLoadingTeams || isSwitchingTeam || loadingMembers || isLoadingGroups;
const contextValue = {
myTeams,
refetchTeams,
isLoading,
onSwitchTeam,
- orgs,
- refetchOrgs,
// create | update team
setEditTeamData,
diff --git a/projects/app/src/web/support/user/team/org/api.ts b/projects/app/src/web/support/user/team/org/api.ts
index c33a5ced9..825ec5446 100644
--- a/projects/app/src/web/support/user/team/org/api.ts
+++ b/projects/app/src/web/support/user/team/org/api.ts
@@ -4,13 +4,14 @@ import type {
putUpdateOrgData,
putUpdateOrgMembersData
} from '@fastgpt/global/support/user/team/org/api';
-import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import type { OrgListItemType } from '@fastgpt/global/support/user/team/org/type';
import type { putMoveOrgType } from '@fastgpt/global/support/user/team/org/api';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
+import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
-export const getOrgList = (path: string) =>
- GET(`/proApi/support/user/team/org/list`, { orgPath: path });
+export const getOrgList = (parentId: ParentIdType) =>
+ GET(`/proApi/support/user/team/org/list`, { parentId });
export const postCreateOrg = (data: postCreateOrgData) =>
POST('/proApi/support/user/team/org/create', data);
diff --git a/projects/app/src/web/support/user/useUserStore.ts b/projects/app/src/web/support/user/useUserStore.ts
index a37715fac..d0247f425 100644
--- a/projects/app/src/web/support/user/useUserStore.ts
+++ b/projects/app/src/web/support/user/useUserStore.ts
@@ -9,7 +9,6 @@ import type { UserType } from '@fastgpt/global/support/user/type.d';
import type { FeTeamPlanStatusType } from '@fastgpt/global/support/wallet/sub/type';
import { getTeamPlanStatus } from './team/api';
import { getGroupList } from './team/group/api';
-import { getOrgList } from './team/org/api';
type State = {
systemMsgReadId: string;
From 37b4a1919bcc9a1e4c34c0bd6aecfa3a383bf858 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Tue, 25 Mar 2025 17:44:38 +0800
Subject: [PATCH 19/35] perf: text splitter (#4313)
* sync collection
* remove lock
* perf: text splitter
* update comment
---
.../zh-cn/docs/development/upgrading/492.md | 7 +-
packages/global/common/string/textSplitter.ts | 56 +-
packages/service/core/ai/audio/speech.ts | 4 +-
packages/service/core/ai/utils.ts | 1 +
.../account/team/OrgManage/index.tsx | 4 +-
.../api/core/dataset/file/getPreviewChunks.ts | 2 +-
.../service/core/dataset/data/controller.ts | 29 +-
.../global/common/string/textSplitter.test.ts | 683 +++++++++++++++++-
8 files changed, 716 insertions(+), 70 deletions(-)
diff --git a/docSite/content/zh-cn/docs/development/upgrading/492.md b/docSite/content/zh-cn/docs/development/upgrading/492.md
index 06999a3eb..37950109f 100644
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
@@ -30,10 +30,15 @@ weight: 799
6. 工作流节点数组字符串类型,自动适配 string 输入。
7. 工作流节点数组类型,自动进行 JSON parse 解析 string 输入。
8. AI proxy 日志优化,去除重试失败的日志,仅保留最后一份错误日志。
+9. 分块算法小调整:
+ * 跨处理符号之间连续性更强。
+ * 代码块分割时,用 LLM 模型上下文作为分块大小,尽可能保证代码块完整性。
+ * 表格分割时,用 LLM 模型上下文作为分块大小,尽可能保证表格完整性。
## 🐛 修复
1. 飞书和语雀知识库无法同步。
2. 渠道测试时,如果配置了模型自定义请求地址,会走自定义请求地址,而不是渠道请求地址。
3. 语音识别模型测试未启用的模型时,无法正常测试。
-4. 管理员配置系统插件时,如果插件包含其他系统应用,无法正常鉴权。
\ No newline at end of file
+4. 管理员配置系统插件时,如果插件包含其他系统应用,无法正常鉴权。
+5. 移除 TTS 自定义请求地址时,必须需要填 requestAuth 字段。
\ No newline at end of file
diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts
index c0b335af7..32863dbb6 100644
--- a/packages/global/common/string/textSplitter.ts
+++ b/packages/global/common/string/textSplitter.ts
@@ -134,8 +134,11 @@ const commonSplit = (props: SplitProps): SplitResponse => {
{ reg: /^(####\s[^\n]+\n)/gm, maxLen: chunkSize },
{ reg: /^(#####\s[^\n]+\n)/gm, maxLen: chunkSize },
- { reg: /([\n]([`~]))/g, maxLen: chunkSize }, // code block
- { reg: /([\n](?=\s*[0-9]+\.))/g, maxLen: chunkSize }, // 增大块,尽可能保证它是一个完整的段落。 (?![\*\-|>`0-9]): markdown special char
+ { reg: /([\n](```[\s\S]*?```|~~~[\s\S]*?~~~))/g, maxLen: maxSize }, // code block
+ {
+ reg: /(\n\|(?:(?:[^\n|]+\|){1,})\n\|(?:[:\-\s]+\|){1,}\n(?:\|(?:[^\n|]+\|)*\n)*)/g,
+ maxLen: maxSize
+ }, // Table 尽可能保证完整性
{ reg: /(\n{2,})/g, maxLen: chunkSize },
{ reg: /([\n])/g, maxLen: chunkSize },
// ------ There's no overlap on the top
@@ -150,7 +153,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
const checkIsCustomStep = (step: number) => step < customRegLen;
const checkIsMarkdownSplit = (step: number) =>
step >= customRegLen && step <= markdownIndex + customRegLen;
- +customReg.length;
+
const checkForbidOverlap = (step: number) => step <= forbidOverlapIndex + customRegLen;
// if use markdown title split, Separate record title
@@ -159,7 +162,8 @@ const commonSplit = (props: SplitProps): SplitResponse => {
return [
{
text,
- title: ''
+ title: '',
+ chunkMaxSize: chunkSize
}
];
}
@@ -167,7 +171,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
const isCustomStep = checkIsCustomStep(step);
const isMarkdownSplit = checkIsMarkdownSplit(step);
- const { reg } = stepReges[step];
+ const { reg, maxLen } = stepReges[step];
const replaceText = (() => {
if (typeof reg === 'string') {
@@ -194,15 +198,19 @@ const commonSplit = (props: SplitProps): SplitResponse => {
})()
);
})();
+
const splitTexts = replaceText.split(splitMarker).filter((part) => part.trim());
return splitTexts
.map((text) => {
const matchTitle = isMarkdownSplit ? text.match(reg)?.[0] || '' : '';
+ // 如果一个分块没有匹配到,则使用默认块大小,否则使用最大块大小
+ const chunkMaxSize = text.match(reg) === null ? chunkSize : maxLen;
return {
text: isMarkdownSplit ? text.replace(matchTitle, '') : text,
- title: matchTitle
+ title: matchTitle,
+ chunkMaxSize
};
})
.filter((item) => !!item.title || !!item.text?.trim());
@@ -252,9 +260,9 @@ const commonSplit = (props: SplitProps): SplitResponse => {
const isCustomStep = checkIsCustomStep(step);
const forbidConcat = isCustomStep; // forbid=true时候,lastText肯定为空
- // oversize
+ // Over step
if (step >= stepReges.length) {
- if (text.length < chunkSize * 3) {
+ if (text.length < maxSize) {
return [text];
}
// use slice-chunkSize to split text
@@ -268,19 +276,18 @@ const commonSplit = (props: SplitProps): SplitResponse => {
// split text by special char
const splitTexts = getSplitTexts({ text, step });
- const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkSize;
- const minChunkLen = chunkSize * 0.7;
-
const chunks: string[] = [];
for (let i = 0; i < splitTexts.length; i++) {
const item = splitTexts[i];
+ const maxLen = item.chunkMaxSize; // 当前块最大长度
+
const lastTextLen = lastText.length;
const currentText = item.text;
const newText = lastText + currentText;
const newTextLen = newText.length;
- // Markdown 模式下,会强制向下拆分最小块,并再最后一个标题时候,给小块都补充上所有标题(包含父级标题)
+ // Markdown 模式下,会强制向下拆分最小块,并再最后一个标题深度,给小块都补充上所有标题(包含父级标题)
if (isMarkdownStep) {
// split new Text, split chunks must will greater 1 (small lastText)
const innerChunks = splitTextRecursively({
@@ -290,11 +297,13 @@ const commonSplit = (props: SplitProps): SplitResponse => {
parentTitle: parentTitle + item.title
});
+ // 只有标题,没有内容。
if (innerChunks.length === 0) {
chunks.push(`${parentTitle}${item.title}`);
continue;
}
+ // 在合并最深级标题时,需要补充标题
chunks.push(
...innerChunks.map(
(chunk) =>
@@ -307,7 +316,16 @@ const commonSplit = (props: SplitProps): SplitResponse => {
// newText is too large(now, The lastText must be smaller than chunkSize)
if (newTextLen > maxLen) {
- // lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
+ const minChunkLen = maxLen * 0.8; // 当前块最小长度
+ const maxChunkLen = maxLen * 1.2; // 当前块最大长度
+
+ // 新文本没有非常大,直接认为它是一个新的块
+ if (newTextLen < maxChunkLen) {
+ chunks.push(newText);
+ lastText = getOneTextOverlapText({ text: newText, step }); // next chunk will start with overlayText
+ continue;
+ }
+ // 上一个文本块已经挺大的,单独做一个块
if (lastTextLen > minChunkLen) {
chunks.push(lastText);
@@ -317,13 +335,13 @@ const commonSplit = (props: SplitProps): SplitResponse => {
continue;
}
- // 说明是新的文本块比较大,需要进一步拆分
+ // 说明是当前文本比较大,需要进一步拆分
- // split new Text, split chunks must will greater 1 (small lastText)
+ // 把新的文本块进行一个拆分,并追加到 latestText 中
const innerChunks = splitTextRecursively({
- text: newText,
+ text: currentText,
step: step + 1,
- lastText: '',
+ lastText,
parentTitle: parentTitle + item.title
});
const lastChunk = innerChunks[innerChunks.length - 1];
@@ -351,11 +369,11 @@ const commonSplit = (props: SplitProps): SplitResponse => {
// Not overlap
if (forbidConcat) {
- chunks.push(item.text);
+ chunks.push(currentText);
continue;
}
- lastText += item.text;
+ lastText = newText;
}
/* If the last chunk is independent, it needs to be push chunks. */
diff --git a/packages/service/core/ai/audio/speech.ts b/packages/service/core/ai/audio/speech.ts
index 0bacaae4e..bb270cee4 100644
--- a/packages/service/core/ai/audio/speech.ts
+++ b/packages/service/core/ai/audio/speech.ts
@@ -30,11 +30,11 @@ export async function text2Speech({
response_format: 'mp3',
speed
},
- modelData.requestUrl && modelData.requestAuth
+ modelData.requestUrl
? {
path: modelData.requestUrl,
headers: {
- Authorization: `Bearer ${modelData.requestAuth}`
+ ...(modelData.requestAuth ? { Authorization: `Bearer ${modelData.requestAuth}` } : {})
}
}
: {}
diff --git a/packages/service/core/ai/utils.ts b/packages/service/core/ai/utils.ts
index 43fdda3bb..6b950b2e3 100644
--- a/packages/service/core/ai/utils.ts
+++ b/packages/service/core/ai/utils.ts
@@ -65,6 +65,7 @@ export const llmCompletionsBodyFormat = (
const requestBody: T = {
...body,
+ model: modelData.model,
temperature:
typeof body.temperature === 'number'
? computedTemperature({
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
index 1c7fe5e53..4c831164d 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
@@ -196,7 +196,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
isLoading={isLoadingOrgs}
>
-
+
@@ -420,7 +420,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
setManageMemberOrg(currentOrg ?? rootOrg)}
+ onClick={() => setManageMemberOrg(currentOrg)}
/>
{currentOrg && currentOrg?.path !== '' && (
<>
diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
index dfa34727f..d27b048f9 100644
--- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
+++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
@@ -94,7 +94,7 @@ async function handler(
per: WritePermissionVal
});
- if (fileAuthRes && (String(fileAuthRes.tmbId) !== String(tmbId) || !fileAuthRes.isRoot)) {
+ if (fileAuthRes && String(fileAuthRes.tmbId) !== String(tmbId) && !fileAuthRes.isRoot) {
return Promise.reject(CommonErrEnum.unAuthFile);
}
diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts
index 3867b56e1..f860d4c87 100644
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -15,18 +15,19 @@ import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTex
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
-import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils';
const formatIndexes = async ({
indexes,
q,
a = '',
- indexSize
+ indexSize,
+ maxIndexSize
}: {
indexes?: (Omit & { dataId?: string })[];
q: string;
a?: string;
indexSize: number;
+ maxIndexSize: number;
}): Promise<
{
type: `${DatasetDataIndexTypeEnum}`;
@@ -46,9 +47,12 @@ const formatIndexes = async ({
}) => {
const qChunks = splitText2Chunks({
text: q,
- chunkSize: indexSize
+ chunkSize: indexSize,
+ maxSize: maxIndexSize
}).chunks;
- const aChunks = a ? splitText2Chunks({ text: a, chunkSize: indexSize }).chunks : [];
+ const aChunks = a
+ ? splitText2Chunks({ text: a, chunkSize: indexSize, maxSize: maxIndexSize }).chunks
+ : [];
return [
...qChunks.map((text) => ({
@@ -100,7 +104,11 @@ const formatIndexes = async ({
// If oversize tokens, split it
const tokens = await countPromptTokens(item.text);
if (tokens > indexSize) {
- const splitText = splitText2Chunks({ text: item.text, chunkSize: 512 }).chunks;
+ const splitText = splitText2Chunks({
+ text: item.text,
+ chunkSize: 512,
+ maxSize: maxIndexSize
+ }).chunks;
return splitText.map((text) => ({
text,
type: item.type
@@ -151,7 +159,8 @@ export async function insertData2Dataset({
indexes,
q,
a,
- indexSize
+ indexSize,
+ maxIndexSize: embModel.maxToken
});
// insert to vector store
@@ -236,7 +245,13 @@ export async function updateData2Dataset({
if (!mongoData) return Promise.reject('core.dataset.error.Data not found');
// 2. Compute indexes
- const formatIndexesResult = await formatIndexes({ indexes, q, a, indexSize });
+ const formatIndexesResult = await formatIndexes({
+ indexes,
+ q,
+ a,
+ indexSize,
+ maxIndexSize: getEmbeddingModel(model).maxToken
+ });
// 3. Patch indexes, create, update, delete
const patchResult: PatchIndexesProps[] = [];
diff --git a/test/cases/function/packages/global/common/string/textSplitter.test.ts b/test/cases/function/packages/global/common/string/textSplitter.test.ts
index 0407cf4da..3a78be5b0 100644
--- a/test/cases/function/packages/global/common/string/textSplitter.test.ts
+++ b/test/cases/function/packages/global/common/string/textSplitter.test.ts
@@ -1,8 +1,13 @@
import { it, expect } from 'vitest'; // 必须显式导入
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
+import * as fs from 'fs';
+
+const simpleChunks = (chunks: string[]) => {
+ return chunks.map((chunk) => chunk.replace(/\s+/g, ''));
+};
// 简单的嵌套测试
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 1`, () => {
const mock = {
text: `# A
@@ -56,7 +61,7 @@ dsgsgfsgs22sddddddd`
const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 2000 });
expect(chunks).toEqual(mock.result);
});
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 2`, () => {
const mock = {
text: `# A
@@ -81,7 +86,7 @@ dsgsgfsgs22`
});
// 普通文本测试:单段不超过 500 字符
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 3`, () => {
const mock = {
text: `快速了解 FastGPT
FastGPT 的能力与优势
@@ -232,13 +237,16 @@ FastGPT 在线使用:https://tryfastgpt.ai
FastGPT 能力
1. 专属 AI 客服
通过导入文档或已有问答对进行训练,让 AI 模型能根据你的文档以交互式对话方式回答问题。
+
2. 简单易用的可视化界面
FastGPT 采用直观的可视化界面设计,为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤,可以轻松完成 AI 客服的创建和训练流程。
+
3. 自动数据预处理
-提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。`,
- `4. 工作流编排
-基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。
-5. 强大的 API 集成
+提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。
+
+4. 工作流编排
+基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。`,
+ `5. 强大的 API 集成
FastGPT 对外的 API 接口对齐了 OpenAI 官方接口,可以直接接入现有的 GPT 应用,也可以轻松集成到企业微信、公众号、飞书等平台。
FastGPT 特点
@@ -262,16 +270,16 @@ FastGPT 遵循附加条件 Apache License 2.0 开源协议,你可以 Fork 之
提供搜索测试、引用修改、完整对话预览等多种调试途径。
-支持多种模型`,
- `支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
+支持多种模型
+
+支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
知识库核心流程
FastGPT AI 相关参数配置说明
-在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。
-
-返回AI内容(高级编排特有)
+在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。`,
+ `返回AI内容(高级编排特有)
这是一个开关,打开的时候,当 AI 对话模块运行时,会将其输出的内容返回到浏览器(API响应);如果关闭,AI 输出的内容不会返回到浏览器,但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。
最大上下文
@@ -290,23 +298,27 @@ FastGPT AI 相关参数配置说明
被放置在上下文数组的最前面,role 为 system,用于引导模型。
引用模板 & 引用提示词
-这两个参数与知识库问答场景相关,可以控制知识库相关的提示词。`,
- `AI 对话消息组成
+这两个参数与知识库问答场景相关,可以控制知识库相关的提示词。
+
+AI 对话消息组成
想使用明白这两个变量,首先要了解传递传递给 AI 模型的消息格式。它是一个数组,FastGPT 中这个数组的组成形式为:
+
[
内置提示词(config.json 配置,一般为空)
系统提示词 (用户输入的提示词)
历史记录
问题(由引用提示词、引用模板和用户问题组成)
]
-🍅
-Tips: 可以通过点击上下文按键查看完整的上下文组成,便于调试。
+🍅`,
+ `Tips: 可以通过点击上下文按键查看完整的上下文组成,便于调试。
+
引用模板和提示词设计
简易模式已移除该功能,仅在工作流中可配置,可点击工作流中AI对话节点内,知识库引用旁边的setting icon进行配置。随着模型的增强,这部分功能将逐步弱化。
+
引用模板和引用提示词通常是成对出现,引用提示词依赖引用模板。
-FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变量)的格式存储,在转义成字符串时候会根据引用模板来进行格式化。`,
- `引用模板和引用提示词通常是成对出现,引用提示词依赖引用模板。
+
FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变量)的格式存储,在转义成字符串时候会根据引用模板来进行格式化。知识库包含多个可用变量: q, a, sourceId(数据的ID), index(第n个数据), source(数据的集合名、文件名),score(距离得分,0-1) 可以通过 {{q}} {{a}} {{sourceId}} {{index}} {{source}} {{score}} 按需引入。下面一个模板例子:
+
可以通过 知识库结构讲解 了解详细的知识库的结构。
引用模板
@@ -324,14 +336,13 @@ FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变
"""
{{quote}}
"""
-对话要求:`,
- `1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
+对话要求:
+1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
2. 使用背景知识回答问题。
3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
我的问题是:"{{question}}"
-转义后则为:
-
-你的背景知识:
+转义后则为:`,
+ `你的背景知识:
"""
{instruction:"电影《铃芽之旅》的导演是谁?",output:"电影《铃芽之旅》的导演是新海诚。",source:"手动输入"}
{instruction:"本作的主人公是谁?",output:"本作的主人公是名叫铃芽的少女。",source:""}
@@ -339,8 +350,8 @@ FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变
"""
对话要求:
1. 背景知识是最新的,其中 instruction 是相关介绍,output 是预期回答或补充。
-2. 使用背景知识回答问题。`,
- `3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
+2. 使用背景知识回答问题。
+3. 背景知识无法回答问题时,你可以礼貌的的回答用户问题。
我的问题是:"{{question}}"
总结
引用模板规定了搜索出来的内容如何组成一句话,其由 q,a,index,source 多个变量组成。
@@ -355,23 +366,22 @@ FastGPT 知识库采用 QA 对(不一定都是问答格式,仅代表两个变
const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 500 });
- const normalizedChunks = chunks.map((chunk) => chunk.replace(/\n/g, ''));
- const normalizedExpected = mock.result.map((result) => result.replace(/\n/g, ''));
+ const normalizedChunks = simpleChunks(chunks);
+ const normalizedExpected = simpleChunks(mock.result);
expect(normalizedChunks).toEqual(normalizedExpected);
});
// 普通文本测试:单段超过 500 字符,有 20% 重叠
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 4`, () => {
const mock = {
text: `FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`,
result: [
- `FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。`,
- `其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`,
- `其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。`,
- `可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`,
- `其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。`,
- `可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`
+ `FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。`,
+ `FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。`,
+ `其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`,
+ `FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。其支持的多种LLM模型使得用户可以根据不同的应用场景选择最合适的模型进行问答,而未来的自定义向量模型支持则为用户提供了更多的可能性。FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。`,
+ `FastGPT的API集成能力不仅体现在与现有GPT应用的无缝对接上,还可以轻松集成到企业微信、公众号、飞书等平台,使得用户可以在不同的平台上享受到FastGPT带来的便利。其开源项目的开放性使得用户可以根据自身需求进行二次开发和发布,从而实现个性化的问答服务。FastGPT的QA结构设计独具匠心,通过对问答流程的优化,提高了在大量数据场景中的问答准确性。可视化工作流的引入则使得用户可以直观地看到从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT提供的多种调试途径,如搜索测试、引用修改和完整对话预览等,使得用户可以在使用过程中随时进行调试,确保问答的准确性和效率。其知识库核心流程中的AI相关参数配置说明详细介绍了如何通过配置来优化AI模型的表现,使得用户可以根据具体需求进行灵活配置。FastGPT的引用模板和提示词设计则为用户提供了更多的定制化选项,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。`
]
};
@@ -381,7 +391,7 @@ it(`Test splitText2Chunks`, () => {
});
// 自定义分隔符测试:分割后,内容少
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 5`, () => {
const mock = {
text: `这是测试文本 1
----
@@ -407,7 +417,7 @@ it(`Test splitText2Chunks`, () => {
});
// 自定义分隔符测试:分割后,内容超长,二次分割,有重叠。
-it(`Test splitText2Chunks`, () => {
+it(`Test splitText2Chunks 6`, () => {
const mock = {
text: `这是测试文本 1,短的
----
@@ -416,8 +426,8 @@ FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提
result: [
`这是测试文本 1,短的`,
`这是测试文本 2,长的。
-FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。`,
- `其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`
+FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提供高效、准确的知识库问答服务而设计。它支持多种数据导入方式,包括手动输入、PDF、WORD、Markdown和CSV等格式,能够自动进行数据预处理、向量化和QA分割,大幅提升数据处理效率。FastGPT的可视化界面设计简洁直观,用户可以通过Flow模块进行工作流编排,轻松实现复杂的问答场景。此外,FastGPT支持多种LLM模型,如GPT、Claude、文心一言等,未来还将支持自定义的向量模型。其强大的API集成能力使得用户可以轻松将其接入现有的GPT应用或其他平台,如企业微信、公众号、飞书等。FastGPT的开源项目遵循Apache License 2.0协议,用户可以进行二次开发和发布。其独特的QA结构设计提高了在大量数据场景中的问答准确性,而可视化工作流则展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。`,
+ `FastGPT还提供了多种调试途径,如搜索测试、引用修改和完整对话预览等,方便用户进行调试。其知识库核心流程包括AI相关参数配置说明,如返回AI内容、最大上下文、函数调用、温度、回复上限、系统提示词、引用模板和引用提示词等。通过这些配置,用户可以更好地控制和优化AI模型的表现。FastGPT的引用模板和提示词设计使得用户可以根据具体需求进行灵活配置,从而提高问答的准确性和效率。总之,FastGPT是一款功能强大、易于使用的智能问答系统,适用于各种应用场景,帮助用户实现高效、准确的知识库问答服务。FastGPT的设计理念是通过先进的技术手段简化用户的操作流程,使得即便是没有技术背景的用户也能轻松上手。其多样化的数据导入方式确保了用户可以根据自身需求选择最合适的方式进行数据输入,而自动化的数据预处理功能则大大减少了用户的工作量。通过对数据的向量化处理,FastGPT能够更好地理解和分析用户的问题,从而提供更为精准的回答。`
]
};
@@ -425,3 +435,600 @@ FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提
expect(chunks).toEqual(mock.result);
});
+
+// 长代码块分割
+it(`Test splitText2Chunks 7`, () => {
+ const mock = {
+ text: `这是一个测试的内容,包含代码块
+
+快速了解 FastGPT
+FastGPT 的能力与优势
+
+FastGPT 是一个基于 LLM 大语言模型的知识库问答系统,提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的问答场景!
+
+FastGPT 在线使用:https://tryfastgpt.ai
+
+FastGPT 能力
+1. 专属 AI 客服
+通过导入文档或已有问答对进行训练,让 AI 模型能根据你的文档以交互式对话方式回答问题。
+
+2. 简单易用的可视化界面
+FastGPT 采用直观的可视化界面设计,为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤,可以轻松完成 AI 客服的创建和训练流程。
+
+~~~js
+import { defaultMaxChunkSize } from '../../core/dataset/training/utils';
+import { getErrText } from '../error/utils';
+
+const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+~~~
+
+3. 自动数据预处理
+提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。
+
+4. 工作流编排
+基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。
+
+5. 强大的 API 集成
+FastGPT 对外的 API 接口对齐了 OpenAI 官方接口,可以直接接入现有的 GPT 应用,也可以轻松集成到企业微信、公众号、飞书等平台。
+
+FastGPT 特点
+项目开源
+
+FastGPT 遵循附加条件 Apache License 2.0 开源协议,你可以 Fork 之后进行二次开发和发布。FastGPT 社区版将保留核心功能,商业版仅在社区版基础上使用 API 的形式进行扩展,不影响学习使用。
+
+独特的 QA 结构
+
+针对客服问答场景设计的 QA 结构,提高在大量数据场景中的问答准确性。
+
+可视化工作流
+
+通过 Flow 模块展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。
+
+无限扩展
+
+基于 API 进行扩展,无需修改 FastGPT 源码,也可快速接入现有的程序中。
+
+便于调试
+
+提供搜索测试、引用修改、完整对话预览等多种调试途径。
+
+支持多种模型
+
+支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
+
+知识库核心流程
+
+FastGPT AI 相关参数配置说明
+
+在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。
+
+返回AI内容(高级编排特有)
+这是一个开关,打开的时候,当 AI 对话模块运行时,会将其输出的内容返回到浏览器(API响应);如果关闭,AI 输出的内容不会返回到浏览器,但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。
+
+最大上下文
+代表模型最多容纳的文字数量。`,
+ result: [
+ `这是一个测试的内容,包含代码块
+
+快速了解 FastGPT
+FastGPT 的能力与优势
+
+FastGPT 是一个基于 LLM 大语言模型的知识库问答系统,提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的问答场景!
+
+FastGPT 在线使用:https://tryfastgpt.ai
+
+FastGPT 能力
+1. 专属 AI 客服
+通过导入文档或已有问答对进行训练,让 AI 模型能根据你的文档以交互式对话方式回答问题。
+
+2. 简单易用的可视化界面
+FastGPT 采用直观的可视化界面设计,为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤,可以轻松完成 AI 客服的创建和训练流程。
+
+~~~js
+import { defaultMaxChunkSize } from '../../core/dataset/training/utils';
+import { getErrText } from '../error/utils';
+
+const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+
+ const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+ const forbidOverlap = checkForbidOverlap(step);
+ const maxOverlapLen = chunkSize * 0.4;
+
+ // step >= stepReges.length: Do not overlap incomplete sentences
+ if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+ const splitTexts = getSplitTexts({ text, step });
+ let overlayText = '';
+
+ for (let i = splitTexts.length - 1; i >= 0; i--) {
+ const currentText = splitTexts[i].text;
+ const newText = currentText + overlayText;
+ const newTextLen = newText.length;
+
+ if (newTextLen > overlapLen) {
+ if (newTextLen > maxOverlapLen) {
+ const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+ return text || overlayText;
+ }
+ return newText;
+ }
+
+ overlayText = newText;
+ }
+ return overlayText;
+ };
+~~~`,
+ `3. 自动数据预处理
+提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径,其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割,节省手动训练时间,提升效能。
+
+4. 工作流编排
+基于 Flow 模块的工作流编排,可以帮助你设计更加复杂的问答流程。例如查询数据库、查询库存、预约实验室等。
+
+5. 强大的 API 集成
+FastGPT 对外的 API 接口对齐了 OpenAI 官方接口,可以直接接入现有的 GPT 应用,也可以轻松集成到企业微信、公众号、飞书等平台。
+
+FastGPT 特点
+项目开源
+
+FastGPT 遵循附加条件 Apache License 2.0 开源协议,你可以 Fork 之后进行二次开发和发布。FastGPT 社区版将保留核心功能,商业版仅在社区版基础上使用 API 的形式进行扩展,不影响学习使用。
+
+独特的 QA 结构
+
+针对客服问答场景设计的 QA 结构,提高在大量数据场景中的问答准确性。
+
+可视化工作流
+
+通过 Flow 模块展示了从问题输入到模型输出的完整流程,便于调试和设计复杂流程。
+`,
+ `无限扩展
+
+基于 API 进行扩展,无需修改 FastGPT 源码,也可快速接入现有的程序中。
+
+便于调试
+
+提供搜索测试、引用修改、完整对话预览等多种调试途径。
+
+支持多种模型
+
+支持 GPT、Claude、文心一言等多种 LLM 模型,未来也将支持自定义的向量模型。
+
+知识库核心流程
+
+FastGPT AI 相关参数配置说明
+
+在 FastGPT 的 AI 对话模块中,有一个 AI 高级配置,里面包含了 AI 模型的参数配置,本文详细介绍这些配置的含义。
+
+返回AI内容(高级编排特有)
+这是一个开关,打开的时候,当 AI 对话模块运行时,会将其输出的内容返回到浏览器(API响应);如果关闭,AI 输出的内容不会返回到浏览器,但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。
+
+最大上下文
+代表模型最多容纳的文字数量。`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({
+ text: mock.text,
+ chunkSize: 500,
+ maxSize: 100000,
+ customReg: ['----']
+ });
+
+ const normalizedChunks = simpleChunks(chunks);
+ const normalizedExpected = simpleChunks(mock.result);
+
+ expect(normalizedChunks).toEqual(normalizedExpected);
+});
+
+// 表格分割测试 - 不超出maxSize
+it(`Test splitText2Chunks 1`, () => {
+ const mock = {
+ text: `测试的呀
+
+| 序号 | 姓名 | 年龄 | 职业 | 城市 |
+|------|------|------|------|------|
+| 1 | 张三 | 25 | 工程师 | 北京 |
+| 2 | 李四 | 30 | 教师 | 上海 |
+| 3 | 王五 | 28 | 医生 | 广州 |
+| 4 | 赵六 | 35 | 律师 | 深圳 |
+| 5 | 孙七 | 27 | 设计师 | 杭州 |
+| 6 | 周八 | 32 | 会计 | 成都 |
+| 7 | 吴九 | 29 | 销售 | 武汉 |
+| 8 | 郑十 | 31 | 记者 | 南京 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+
+这是第二段了`,
+ result: [
+ `测试的呀
+
+| 序号 | 姓名 | 年龄 | 职业 | 城市 |
+|------|------|------|------|------|
+| 1 | 张三 | 25 | 工程师 | 北京 |
+| 2 | 李四 | 30 | 教师 | 上海 |
+| 3 | 王五 | 28 | 医生 | 广州 |
+| 4 | 赵六 | 35 | 律师 | 深圳 |
+| 5 | 孙七 | 27 | 设计师 | 杭州 |
+| 6 | 周八 | 32 | 会计 | 成都 |
+| 7 | 吴九 | 29 | 销售 | 武汉 |
+| 8 | 郑十 | 31 | 记者 | 南京 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+
+这是第二段了`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 2000 });
+ expect(chunks).toEqual(mock.result);
+});
+// 表格分割测试 - 超出maxSize
+it(`Test splitText2Chunks 1`, () => {
+ const mock = {
+ text: `测试的呀
+
+| 序号 | 姓名 | 年龄 | 职业 | 城市 |
+|------|------|------|------|------|
+| 1 | 张三 | 25 | 工程师 | 北京 |
+| 2 | 李四 | 30 | 教师 | 上海 |
+| 3 | 王五 | 28 | 医生 | 广州 |
+| 4 | 赵六 | 35 | 律师 | 深圳 |
+| 5 | 孙七 | 27 | 设计师 | 杭州 |
+| 6 | 周八 | 32 | 会计 | 成都 |
+| 7 | 吴九 | 29 | 销售 | 武汉 |
+| 8 | 郑十 | 31 | 记者 | 南京 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+
+这是第二段了`,
+ result: [
+ `测试的呀
+
+| 序号 | 姓名 | 年龄 | 职业 | 城市 |
+|------|------|------|------|------|
+| 1 | 张三 | 25 | 工程师 | 北京 |
+| 2 | 李四 | 30 | 教师 | 上海 |
+| 3 | 王五 | 28 | 医生 | 广州 |
+| 4 | 赵六 | 35 | 律师 | 深圳 |
+| 5 | 孙七 | 27 | 设计师 | 杭州 |
+| 6 | 周八 | 32 | 会计 | 成都 |
+| 7 | 吴九 | 29 | 销售 | 武汉 |
+| 8 | 郑十 | 31 | 记者 | 南京 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |`,
+ `| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 9 | 刘一 | 33 | 建筑师 | 天津 |
+| 10 | 陈二 | 26 | 程序员 | 重庆 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1001 | 杨一 | 34 | 程序员 | 厦门 |
+| 1002 | 杨二 | 34 | 程序员 | 厦门 |
+| 1003 | 杨三 | 34 | 程序员 | 厦门 |
+| 1004 | 杨四 | 34 | 程序员 | 厦门 |
+| 1005 | 杨五 | 34 | 程序员 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+| 1000 | 黄末 | 28 | 作家 | 厦门 |
+
+这是第二段了`
+ ]
+ };
+
+ const { chunks } = splitText2Chunks({ text: mock.text, chunkSize: 512, maxSize: 512 });
+
+ expect(chunks).toEqual(mock.result);
+});
From ff64a3c03909b1b93720931cdfb60ab8a664eded Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Tue, 25 Mar 2025 18:06:31 +0800
Subject: [PATCH 20/35] update search filter code (#4317)
* sync collection
* remove lock
* update search filter code
---
packages/service/core/dataset/search/controller.ts | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts
index 9e87bf9bc..9f59aba8d 100644
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -134,12 +134,10 @@ export const filterDatasetDataByMaxTokens = async (
let totalTokens = 0;
for await (const item of tokensScoreFilter) {
+ results.push(item);
+
totalTokens += item.tokens;
- if (totalTokens > maxTokens + 500) {
- break;
- }
- results.push(item);
if (totalTokens > maxTokens) {
break;
}
From 1fdf947a13e9f285f91d14700593079d4920b189 Mon Sep 17 00:00:00 2001
From: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Date: Tue, 25 Mar 2025 21:08:51 +0800
Subject: [PATCH 21/35] pref: member/group/org (#4316)
* feat: change group owner api
* pref: member/org/group
* fix: member modal select clb
* fix: search member when change owner
---
.../global/support/user/team/org/api.d.ts | 9 +-
.../global/support/user/team/org/constant.ts | 5 +-
.../global/support/user/team/org/type.d.ts | 2 +-
.../service/support/permission/auth/org.ts | 4 +-
.../components/common/Avatar/AvatarGroup.tsx | 2 +-
.../permission/ChangeOwnerModal/index.tsx | 22 ++-
.../permission/MemberManager/MemberModal.tsx | 183 +++++++-----------
.../team/GroupManage/GroupInfoModal.tsx | 15 +-
.../team/GroupManage/GroupManageMember.tsx | 19 +-
.../GroupManage/GroupTransferOwnerModal.tsx | 33 +++-
.../account/team/GroupManage/index.tsx | 27 ++-
.../account/team/MemberTable.tsx | 47 +++--
.../account/team/OrgManage/OrgInfoModal.tsx | 16 +-
.../team/OrgManage/OrgMemberManageModal.tsx | 95 ++++++---
.../account/team/OrgManage/OrgMoveModal.tsx | 17 +-
.../account/team/OrgManage/OrgTree.tsx | 86 ++++----
.../account/team/OrgManage/index.tsx | 121 ++++--------
.../pageComponents/account/team/context.tsx | 19 +-
.../app/src/web/support/user/team/org/api.ts | 16 +-
.../support/user/team/org/hooks/useOrg.tsx | 108 +++++++++++
20 files changed, 496 insertions(+), 350 deletions(-)
create mode 100644 projects/app/src/web/support/user/team/org/hooks/useOrg.tsx
diff --git a/packages/global/support/user/team/org/api.d.ts b/packages/global/support/user/team/org/api.d.ts
index 69ca09796..feaded6f1 100644
--- a/packages/global/support/user/team/org/api.d.ts
+++ b/packages/global/support/user/team/org/api.d.ts
@@ -1,12 +1,13 @@
+// orgId, pathid, path === null ===> root org
export type postCreateOrgData = {
name: string;
- parentId: string;
description?: string;
avatar?: string;
+ path?: string;
};
export type putUpdateOrgMembersData = {
- orgId: string;
+ orgId?: string;
members: {
tmbId: string;
// role: `${OrgMemberRole}`;
@@ -14,7 +15,7 @@ export type putUpdateOrgMembersData = {
};
export type putUpdateOrgData = {
- orgId: string;
+ orgId: string; // can not be undefined because can not uppdate root org
name?: string;
avatar?: string;
description?: string;
@@ -22,7 +23,7 @@ export type putUpdateOrgData = {
export type putMoveOrgType = {
orgId: string;
- targetOrgId: string;
+ targetOrgId?: string; // '' ===> move to root org
};
// type putChnageOrgOwnerData = {
diff --git a/packages/global/support/user/team/org/constant.ts b/packages/global/support/user/team/org/constant.ts
index c5aea21ab..cc335d930 100644
--- a/packages/global/support/user/team/org/constant.ts
+++ b/packages/global/support/user/team/org/constant.ts
@@ -3,7 +3,10 @@ import { OrgSchemaType } from './type';
export const OrgCollectionName = 'team_orgs';
export const OrgMemberCollectionName = 'team_org_members';
-export const getOrgChildrenPath = (org: OrgSchemaType) => `${org.path}/${org.pathId}`;
+export const getOrgChildrenPath = (org: OrgSchemaType) => {
+ if (org.path === '' && org.pathId === '') return '';
+ return `${org.path ?? ''}/${org.pathId}`;
+};
export enum SyncOrgSourceEnum {
wecom = 'wecom'
diff --git a/packages/global/support/user/team/org/type.d.ts b/packages/global/support/user/team/org/type.d.ts
index f2afac2d5..1f0f45608 100644
--- a/packages/global/support/user/team/org/type.d.ts
+++ b/packages/global/support/user/team/org/type.d.ts
@@ -21,7 +21,7 @@ type OrgMemberSchemaType = {
};
export type OrgListItemType = OrgSchemaType & {
- permission: TeamPermission;
+ permission?: TeamPermission;
total: number; // members + children orgs
};
diff --git a/packages/service/support/permission/auth/org.ts b/packages/service/support/permission/auth/org.ts
index d864b1cb5..a569a18ae 100644
--- a/packages/service/support/permission/auth/org.ts
+++ b/packages/service/support/permission/auth/org.ts
@@ -4,14 +4,14 @@ import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { authUserPer } from '../user/auth';
import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant';
-/*
+/*
Team manager can control org
*/
export const authOrgMember = async ({
orgIds,
...props
}: {
- orgIds: string | string[];
+ orgIds?: string | string[];
} & AuthModeType): Promise => {
const result = await authUserPer({
...props,
diff --git a/packages/web/components/common/Avatar/AvatarGroup.tsx b/packages/web/components/common/Avatar/AvatarGroup.tsx
index 947e488bf..f57ba7fc8 100644
--- a/packages/web/components/common/Avatar/AvatarGroup.tsx
+++ b/packages/web/components/common/Avatar/AvatarGroup.tsx
@@ -19,7 +19,7 @@ function AvatarGroup({
avatars: string[];
total?: number;
}) {
- const remain = total ?? avatars.length - max;
+ const remain = (total ?? avatars.length) - max;
return (
{avatars.slice(0, max).map((avatar, index) => (
diff --git a/projects/app/src/components/support/permission/ChangeOwnerModal/index.tsx b/projects/app/src/components/support/permission/ChangeOwnerModal/index.tsx
index 5818a39d7..a379447fa 100644
--- a/projects/app/src/components/support/permission/ChangeOwnerModal/index.tsx
+++ b/projects/app/src/components/support/permission/ChangeOwnerModal/index.tsx
@@ -1,3 +1,4 @@
+import { GetSearchUserGroupOrg } from '@/web/support/user/api';
import { getTeamMembers } from '@/web/support/user/team/api';
import {
Box,
@@ -38,16 +39,29 @@ export function ChangeOwnerModal({
pageSize: 15
});
- const memberList = teamMembers.filter((item) => {
- return item.memberName.includes(inputValue);
- });
+ const { data: searchedData } = useRequest2(
+ async () => {
+ if (!inputValue) return;
+ return GetSearchUserGroupOrg(inputValue);
+ },
+ {
+ manual: false,
+ refreshDeps: [inputValue],
+ throttleWait: 500,
+ debounceWait: 200
+ }
+ );
+ const memberList = searchedData ? searchedData.members : teamMembers;
const {
isOpen: isOpenMemberListMenu,
onClose: onCloseMemberListMenu,
onOpen: onOpenMemberListMenu
} = useDisclosure();
- const [selectedMember, setSelectedMember] = useState(null);
+ const [selectedMember, setSelectedMember] = useState | null>(null);
const { runAsync, loading } = useRequest2(onChangeOwner, {
onSuccess: onClose,
diff --git a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
index da5468d8f..22e56d470 100644
--- a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
+++ b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
@@ -30,7 +30,7 @@ import {
import Path from '@/components/common/folder/Path';
import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
-import { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import { useContextSelector } from 'use-context-selector';
import { CollaboratorContext } from './context';
import { getTeamMembers } from '@/web/support/user/team/api';
@@ -39,6 +39,9 @@ import { getOrgList, getOrgMembers } from '@/web/support/user/team/org/api';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import MemberItemCard from './MemberItemCard';
import { GetSearchUserGroupOrg } from '@/web/support/user/api';
+import useOrg from '@/web/support/user/team/org/hooks/useOrg';
+import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
const HoverBoxStyle = {
bgColor: 'myGray.50',
@@ -57,48 +60,21 @@ function MemberModal({
const collaboratorList = useContextSelector(CollaboratorContext, (v) => v.collaboratorList);
const [searchText, setSearchText] = useState('');
const [filterClass, setFilterClass] = useState<'member' | 'org' | 'group'>();
- const [path, setPath] = useState('');
- const [orgStack, setOrgStack] = useState([]);
- const currentOrg = useMemo(() => orgStack[orgStack.length - 1], [orgStack]);
+ const {
+ paths,
+ onClickOrg,
+ members: orgMembers,
+ MemberScrollData: OrgMemberScrollData,
+ onPathClick,
+ refresh,
+ updateCurrentOrg,
+ orgs
+ } = useOrg({ getPermission: false });
const { data: members, ScrollData: TeamMemberScrollData } = useScrollPagination(getTeamMembers, {
pageSize: 15
});
- const [rootOrg, setRootOrg] = useState();
- const { data: orgMembers = [], ScrollData: OrgMemberScrollData } = useScrollPagination(
- getOrgMembers,
- {
- pageSize: 20,
- params: {
- orgId: currentOrg?._id ?? rootOrg?._id
- },
- refreshDeps: [currentOrg?._id]
- }
- );
- const onClickOrg = (org: OrgType) => {
- setOrgStack([...orgStack, org]);
- setPath(getOrgChildrenPath(org));
- };
-
- const { data: orgs = [] } = useRequest2(
- () => {
- const splitPath = path.split('/').filter(Boolean);
- const orgs = orgStack.filter((o) => splitPath.includes(o.pathId));
- setOrgStack(orgs);
- return getOrgList(path);
- },
- {
- manual: false,
- refreshDeps: [path],
- onSuccess: (data) => {
- if (!rootOrg) {
- setRootOrg(data[0]);
- }
- }
- }
- );
-
const { data: groups = [], loading: loadingGroupsAndOrgs } = useRequest2(
async () => {
if (!userInfo?.team?.teamId) return [];
@@ -117,21 +93,9 @@ function MemberModal({
refreshDeps: [searchText]
});
- const paths = useMemo(() => {
- return orgStack
- .map((org) => {
- if (org?.path === '') return;
- return {
- parentId: getOrgChildrenPath(org),
- parentName: org.name
- };
- })
- .filter(Boolean) as ParentTreePathItemType[];
- }, [orgStack]);
+ const [selectedOrgList, setSelectedOrgIdList] = useState([]);
- const [selectedOrgIdList, setSelectedOrgIdList] = useState([]);
-
- const filterOrgs: (OrgType & { count?: number })[] = useMemo(() => {
+ const filterOrgs: (OrgListItemType & { count?: number })[] = useMemo(() => {
if (searchText && searchedData) {
const orgids = searchedData.orgs.map((item) => item._id);
return orgs.filter((org) => orgids.includes(String(org._id)));
@@ -144,7 +108,9 @@ function MemberModal({
}));
}, [searchText, orgs, searchedData]);
- const [selectedMemberIdList, setSelectedMembers] = useState([]);
+ const [selectedMemberList, setSelectedMemberList] = useState<
+ Omit[]
+ >([]);
const filterMembers = useMemo(() => {
if (searchText) {
return searchedData?.members || [];
@@ -152,9 +118,8 @@ function MemberModal({
return members;
}, [searchText, members, searchedData?.members]);
- console.log(filterMembers);
- const [selectedGroupIdList, setSelectedGroupIdList] = useState([]);
+ const [selectedGroupList, setSelectedGroupList] = useState([]);
const filterGroups = useMemo(() => {
if (searchText) {
return searchedData?.groups.map((item) => ({
@@ -186,9 +151,9 @@ function MemberModal({
const { runAsync: onConfirm, loading: isUpdating } = useRequest2(
() =>
onUpdateCollaborators({
- members: selectedMemberIdList,
- groups: selectedGroupIdList,
- orgs: selectedOrgIdList,
+ members: selectedMemberList.map((item) => item.tmbId),
+ groups: selectedGroupList.map((item) => item._id),
+ orgs: selectedOrgList.map((item) => item._id),
permission: selectedPermission!
}),
{
@@ -206,42 +171,31 @@ function MemberModal({
]);
const selectedList = useMemo(() => {
- const selectedOrgs = orgs.filter((org) => selectedOrgIdList.includes(org._id));
- const selectedGroups = groups.filter((group) => selectedGroupIdList.includes(group._id));
- const selectedMembers = members.filter((member) => selectedMemberIdList.includes(member.tmbId));
-
return [
- ...selectedOrgs.map((item) => ({
+ ...selectedOrgList.map((item) => ({
id: `org-${item._id}`,
avatar: item.avatar,
name: item.name,
- onDelete: () => setSelectedOrgIdList(selectedOrgIdList.filter((v) => v !== item._id)),
+ onDelete: () => setSelectedOrgIdList(selectedOrgList.filter((v) => v._id !== item._id)),
orgs: undefined
})),
- ...selectedGroups.map((item) => ({
+ ...selectedGroupList.map((item) => ({
id: `group-${item._id}`,
avatar: item.avatar,
name: item.name === DefaultGroupName ? userInfo?.team.teamName : item.name,
- onDelete: () => setSelectedGroupIdList(selectedGroupIdList.filter((v) => v !== item._id)),
+ onDelete: () => setSelectedGroupList(selectedGroupList.filter((v) => v._id !== item._id)),
orgs: undefined
})),
- ...selectedMembers.map((item) => ({
+ ...selectedMemberList.map((item) => ({
id: `member-${item.tmbId}`,
avatar: item.avatar,
name: item.memberName,
- onDelete: () => setSelectedMembers(selectedMemberIdList.filter((v) => v !== item.tmbId)),
+ onDelete: () =>
+ setSelectedMemberList(selectedMemberList.filter((v) => v.tmbId !== item.tmbId)),
orgs: item.orgs
}))
];
- }, [
- orgs,
- groups,
- members,
- selectedOrgIdList,
- selectedGroupIdList,
- selectedMemberIdList,
- userInfo?.team.teamName
- ]);
+ }, [selectedOrgList, selectedGroupList, selectedMemberList, userInfo?.team.teamName]);
return (
{
if (parentId === '') {
setFilterClass(undefined);
- setPath('');
+ onPathClick('');
} else if (
parentId === 'member' ||
parentId === 'org' ||
parentId === 'group'
) {
setFilterClass(parentId);
- setPath('');
+ onPathClick('');
} else {
- setPath(parentId);
+ onPathClick(parentId);
}
}}
rootName={t('common:common.Team')}
/>
)}
- {(filterClass === 'member' || (searchText && filterMembers.length > 0)) && (
-
- {filterMembers?.map((member) => {
+ {(filterClass === 'member' || (searchText && filterMembers.length > 0)) &&
+ (() => {
+ const members = filterMembers?.map((member) => {
const onChange = () => {
- setSelectedMembers((state) => {
- if (state.includes(member.tmbId)) {
- return state.filter((v) => v !== member.tmbId);
+ setSelectedMemberList((state) => {
+ if (state.find((v) => v.tmbId === member.tmbId)) {
+ return state.filter((v) => v.tmbId !== member.tmbId);
}
- return [...state, member.tmbId];
+ return [...state, member];
});
};
const collaborator = collaboratorList?.find((v) => v.tmbId === member.tmbId);
@@ -364,23 +313,33 @@ function MemberModal({
name={member.memberName}
permission={collaborator?.permission.value}
onChange={onChange}
- isChecked={selectedMemberIdList.includes(member.tmbId)}
+ isChecked={!!selectedMemberList.find((v) => v.tmbId === member.tmbId)}
orgs={member.orgs}
/>
);
- })}
-
- )}
-
+ });
+ return searchText ? (
+ members
+ ) : (
+
+ {members}
+
+ );
+ })()}
{(filterClass === 'org' || searchText) &&
(() => {
const orgs = filterOrgs?.map((org) => {
const onChange = () => {
setSelectedOrgIdList((state) => {
- if (state.includes(org._id)) {
- return state.filter((v) => v !== org._id);
+ if (state.find((v) => v._id === org._id)) {
+ return state.filter((v) => v._id !== org._id);
}
- return [...state, org._id];
+ return [...state, org];
});
};
const collaborator = collaboratorList?.find((v) => v.orgId === org._id);
@@ -396,7 +355,7 @@ function MemberModal({
onClick={onChange}
>
v._id === org._id)}
pointerEvents="none"
/>
@@ -442,14 +401,14 @@ function MemberModal({
key={member.tmbId}
name={member.memberName}
onChange={() => {
- setSelectedMembers((state) => {
- if (state.includes(member.tmbId)) {
- return state.filter((v) => v !== member.tmbId);
+ setSelectedMemberList((state) => {
+ if (state.find((v) => v.tmbId === member.tmbId)) {
+ return state.filter((v) => v.tmbId !== member.tmbId);
}
- return [...state, member.tmbId];
+ return [...state, member];
});
}}
- isChecked={selectedMemberIdList.includes(member.tmbId)}
+ isChecked={!!selectedMemberList.find((v) => v.tmbId === member.tmbId)}
orgs={member.orgs}
/>
);
@@ -459,11 +418,11 @@ function MemberModal({
})()}
{filterGroups?.map((group) => {
const onChange = () => {
- setSelectedGroupIdList((state) => {
- if (state.includes(group._id)) {
- return state.filter((v) => v !== group._id);
+ setSelectedGroupList((state) => {
+ if (state.find((v) => v._id === group._id)) {
+ return state.filter((v) => v._id !== group._id);
}
- return [...state, group._id];
+ return [...state, group];
});
};
const collaborator = collaboratorList?.find((v) => v.groupId === group._id);
@@ -476,7 +435,7 @@ function MemberModal({
}
permission={collaborator?.permission.value}
onChange={onChange}
- isChecked={selectedGroupIdList.includes(group._id)}
+ isChecked={!!selectedGroupList.find((v) => v._id === group._id)}
/>
);
})}
@@ -486,7 +445,7 @@ function MemberModal({
{`${t('user:has_chosen')}: `}
- {selectedMemberIdList.length + selectedGroupIdList.length + selectedOrgIdList.length}
+ {selectedMemberList.length + selectedGroupList.length + selectedOrgList.length}
{selectedList.map((item) => {
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
index 9fd81a310..c86134bfe 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
@@ -12,14 +12,25 @@ import { useContextSelector } from 'use-context-selector';
import { TeamContext } from '../context';
import { postCreateGroup, putUpdateGroup } from '@/web/support/user/team/group/api';
import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
+import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
export type GroupFormType = {
avatar: string;
name: string;
};
-function GroupInfoModal({ onClose, editGroupId }: { onClose: () => void; editGroupId?: string }) {
- const { refetchGroups, groups, refetchMembers } = useContextSelector(TeamContext, (v) => v);
+function GroupInfoModal({
+ onClose,
+ editGroupId,
+ groups,
+ refetchGroups
+}: {
+ onClose: () => void;
+ editGroupId?: string;
+ groups: MemberGroupListType;
+ refetchGroups: () => void;
+}) {
+ const { refetchMembers } = useContextSelector(TeamContext, (v) => v);
const { t } = useTranslation();
const {
File: AvatarSelect,
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index e0d1b05f4..7b13fc301 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -24,7 +24,10 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
-import { GroupMemberItemType } from '@fastgpt/global/support/permission/memberGroup/type';
+import {
+ GroupMemberItemType,
+ MemberGroupListType
+} from '@fastgpt/global/support/permission/memberGroup/type';
import { useMount } from 'ahooks';
export type GroupFormType = {
@@ -37,13 +40,21 @@ export type GroupFormType = {
// 1. Owner can not be deleted, toast
// 2. Owner/Admin can manage members
// 3. Owner can add/remove admins
-function GroupEditModal({ onClose, editGroupId }: { onClose: () => void; editGroupId?: string }) {
+function GroupEditModal({
+ onClose,
+ editGroupId,
+ groups,
+ refetchGroups
+}: {
+ onClose: () => void;
+ editGroupId?: string;
+ groups: MemberGroupListType;
+ refetchGroups: () => void;
+}) {
const { t } = useTranslation();
const { userInfo } = useUserStore();
const { toast } = useToast();
- const groups = useContextSelector(TeamContext, (v) => v.groups);
- const refetchGroups = useContextSelector(TeamContext, (v) => v.refetchGroups);
const group = useMemo(() => {
return groups.find((item) => item._id === editGroupId);
}, [editGroupId, groups]);
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
index aab5696b1..93b0bc7cc 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
@@ -18,25 +18,43 @@ import { useTranslation } from 'next-i18next';
import React, { useMemo, useState } from 'react';
import { TeamContext } from '../context';
import { useContextSelector } from 'use-context-selector';
+import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { GetSearchUserGroupOrg } from '@/web/support/user/api';
+import { Omit } from '@fastgpt/web/components/common/DndDrag';
export type ChangeOwnerModalProps = {
groupId: string;
+ groups: MemberGroupListType;
+ refetchGroups: () => void;
};
export function ChangeOwnerModal({
onClose,
- groupId
+ groupId,
+ groups,
+ refetchGroups
}: ChangeOwnerModalProps & { onClose: () => void }) {
const { t } = useTranslation();
const [inputValue, setInputValue] = React.useState('');
- const { members: allMembers, groups, refetchGroups } = useContextSelector(TeamContext, (v) => v);
+ const { data: searchedData } = useRequest2(
+ async () => {
+ if (!inputValue) return;
+ return GetSearchUserGroupOrg(inputValue);
+ },
+ {
+ manual: false,
+ refreshDeps: [inputValue],
+ throttleWait: 500,
+ debounceWait: 200
+ }
+ );
+
+ const { members: allMembers } = useContextSelector(TeamContext, (v) => v);
const group = useMemo(() => {
return groups.find((item) => item._id === groupId);
}, [groupId, groups]);
- const memberList = allMembers.filter((item) => {
- return item.memberName.toLowerCase().includes(inputValue.toLowerCase());
- });
+ const memberList = searchedData ? searchedData.members : allMembers;
const [keepAdmin, setKeepAdmin] = useState(true);
@@ -46,7 +64,10 @@ export function ChangeOwnerModal({
onOpen: onOpenMemberListMenu
} = useDisclosure();
- const [selectedMember, setSelectedMember] = useState(null);
+ const [selectedMember, setSelectedMember] = useState | null>(null);
const { runAsync, loading } = useRequest2(
(tmbId: string) => putGroupChangeOwner(groupId, tmbId),
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
index f411bc3f5..574e52117 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
@@ -22,7 +22,7 @@ import MyMenu, { MenuItemType } from '@fastgpt/web/components/common/MyMenu';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { deleteGroup, getGroupMembers } from '@/web/support/user/team/group/api';
+import { deleteGroup, getGroupList, getGroupMembers } from '@/web/support/user/team/group/api';
import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant';
import MemberTag from '../../../../components/support/user/team/Info/MemberTag';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
@@ -39,9 +39,19 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { userInfo } = useUserStore();
- const { groups, refetchGroups, members, teamSize } = useContextSelector(TeamContext, (v) => v);
+ const { members, teamSize } = useContextSelector(TeamContext, (v) => v);
+
+ const {
+ data: groups = [],
+ loading: isLoadingGroups,
+ refresh: refetchGroups
+ } = useRequest2(getGroupList, {
+ manual: false,
+ refreshDeps: [userInfo?.team?.teamId]
+ });
const [editGroup, setEditGroup] = useState();
+
const {
isOpen: isOpenGroupInfo,
onOpen: onOpenGroupInfo,
@@ -106,7 +116,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
)}
-
+
@@ -230,10 +240,17 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{isOpenChangeOwner && editGroup && (
-
+
)}
{isOpenGroupInfo && (
{
onCloseGroupInfo();
setEditGroup(undefined);
@@ -243,6 +260,8 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
)}
{isOpenManageGroupMember && editGroup && (
{
onCloseManageGroupMember();
setEditGroup(undefined);
diff --git a/projects/app/src/pageComponents/account/team/MemberTable.tsx b/projects/app/src/pageComponents/account/team/MemberTable.tsx
index 994008bc6..edc1eedf4 100644
--- a/projects/app/src/pageComponents/account/team/MemberTable.tsx
+++ b/projects/app/src/pageComponents/account/team/MemberTable.tsx
@@ -44,24 +44,20 @@ import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import { useState } from 'react';
import { downloadFetch } from '@/web/common/system/utils';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
+import { useToast } from '@fastgpt/web/hooks/useToast';
const InviteModal = dynamic(() => import('./Invite/InviteModal'));
const TeamTagModal = dynamic(() => import('@/components/support/user/team/TeamTagModal'));
function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
+ const { toast } = useToast();
+
const { userInfo } = useUserStore();
const { feConfigs } = useSystemStore();
- const {
- refetchGroups,
- myTeams,
- refetchTeams,
- members,
- refetchMembers,
- onSwitchTeam,
- MemberScrollData
- } = useContextSelector(TeamContext, (v) => v);
+ const { myTeams, refetchTeams, members, refetchMembers, onSwitchTeam, MemberScrollData } =
+ useContextSelector(TeamContext, (v) => v);
const {
isOpen: isOpenTeamTagsAsync,
@@ -85,7 +81,10 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const isSyncMember = feConfigs.register_method?.includes('sync');
const { data: searchMembersData } = useRequest2(
- () => GetSearchUserGroupOrg(searchText, { members: true, orgs: false, groups: false }),
+ async () => {
+ if (!searchText) return Promise.resolve();
+ return GetSearchUserGroupOrg(searchText, { members: true, orgs: false, groups: false });
+ },
{
manual: false,
throttleWait: 500,
@@ -137,13 +136,12 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
canEmpty: false,
rows: 1
});
-
const handleEditMemberName = (tmbId: string, memberName: string) => {
openEditMemberName({
defaultVal: memberName,
onSuccess: (newName: string) => {
return putUpdateMemberNameByManager(tmbId, newName).then(() => {
- Promise.all([refetchGroups(), refetchMembers()]);
+ refetchMembers();
});
},
onError: (err) => {
@@ -323,10 +321,27 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
}}
onClick={() => {
openRemoveMember(
- () =>
- delRemoveMember(member.tmbId).then(() =>
- Promise.all([refetchGroups(), refetchMembers()])
- ),
+ () => delRemoveMember(member.tmbId).then(refetchMembers),
+ undefined,
+ t('account_team:remove_tip', {
+ username: member.memberName
+ })
+ )();
+ }}
+ />
+ {
+ openRemoveMember(
+ () => delRemoveMember(member.tmbId).then(refetchMembers),
undefined,
t('account_team:remove_tip', {
username: member.memberName
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
index c992fbead..01d81362d 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx
@@ -14,8 +14,7 @@ export type OrgFormType = {
avatar: string;
description?: string;
name: string;
- path: string;
- parentId?: string;
+ path?: string;
};
export const defaultOrgForm: OrgFormType = {
@@ -29,11 +28,13 @@ export const defaultOrgForm: OrgFormType = {
function OrgInfoModal({
editOrg,
onClose,
- onSuccess
+ onSuccess,
+ updateCurrentOrg
}: {
editOrg: OrgFormType;
onClose: () => void;
onSuccess: () => void;
+ updateCurrentOrg: (data: { name?: string; avatar?: string; description?: string }) => void;
}) {
const { t } = useTranslation();
@@ -50,11 +51,10 @@ function OrgInfoModal({
const { run: onCreate, loading: isLoadingCreate } = useRequest2(
async (data: OrgFormType) => {
- if (!editOrg.parentId) return;
return postCreateOrg({
name: data.name,
avatar: data.avatar,
- parentId: editOrg.parentId,
+ path: editOrg.path,
description: data.description
});
},
@@ -67,7 +67,7 @@ function OrgInfoModal({
}
);
- const { run: onUpdate, loading: isLoadingUpdate } = useRequest2(
+ const { runAsync: onUpdate, loading: isLoadingUpdate } = useRequest2(
async (data: OrgFormType) => {
if (!editOrg._id) return;
return putUpdateOrg({
@@ -144,7 +144,9 @@ function OrgInfoModal({
isLoading={isLoading}
onClick={handleSubmit((data) => {
if (isEdit) {
- onUpdate(data);
+ onUpdate(data).then(() => {
+ updateCurrentOrg(data);
+ });
} else {
onCreate(data);
}
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
index 13a3bb204..aec30c1b0 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
@@ -17,12 +17,11 @@ import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import type React from 'react';
import { useEffect, useMemo, useState } from 'react';
-import { useContextSelector } from 'use-context-selector';
-import { TeamContext } from '../context';
-import { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
+import { OrgListItemType } from '@fastgpt/global/support/user/team/org/type';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
+import { getTeamMembers } from '@/web/support/user/team/api';
export type GroupFormType = {
members: {
@@ -51,20 +50,39 @@ function OrgMemberManageModal({
onClose: () => void;
}) {
const { t } = useTranslation();
- const { members: allMembers, MemberScrollData } = useContextSelector(TeamContext, (v) => v);
- const { data: orgMembers, ScrollData: OrgMemberScrollData } = useScrollPagination(getOrgMembers, {
+
+ const {
+ data: allMembers,
+ ScrollData: MemberScrollData,
+ isLoading: isLoadingMembers
+ } = useScrollPagination(getTeamMembers, {
pageSize: 20,
params: {
- orgId: currentOrg?._id ?? ''
+ withLeaved: false
}
});
- const [selectedMembers, setSelectedMembers] = useState(
- orgMembers.map((item) => item.tmbId)
- );
+ const {
+ data: orgMembers,
+ ScrollData: OrgMemberScrollData,
+ isLoading: isLoadingOrgMembers
+ } = useScrollPagination(getOrgMembers, {
+ pageSize: 20,
+ params: {
+ orgPath: getOrgChildrenPath(currentOrg)
+ }
+ });
+
+ const [selected, setSelected] = useState<{ name: string; tmbId: string; avatar: string }[]>([]);
useEffect(() => {
- setSelectedMembers(orgMembers.map((item) => item.tmbId));
+ setSelected(
+ orgMembers.map((item) => ({
+ name: item.memberName,
+ tmbId: item.tmbId,
+ avatar: item.avatar
+ }))
+ );
}, [orgMembers]);
const [searchKey, setSearchKey] = useState('');
@@ -78,8 +96,8 @@ function OrgMemberManageModal({
() => {
return putUpdateOrgMembers({
orgId: currentOrg._id,
- members: selectedMembers.map((tmbId) => ({
- tmbId
+ members: selected.map((member) => ({
+ tmbId: member.tmbId
}))
});
},
@@ -92,15 +110,25 @@ function OrgMemberManageModal({
}
);
- const isSelected = (memberId: string) => {
- return selectedMembers.find((tmbId) => tmbId === memberId);
+ const isSelected = (tmbId: string) => {
+ return selected.find((tmb) => tmb.tmbId === tmbId);
};
- const handleToggleSelect = (memberId: string) => {
- if (isSelected(memberId)) {
- setSelectedMembers((state) => state.filter((tmbId) => tmbId !== memberId));
+ const handleToggleSelect = (tmbId: string) => {
+ if (isSelected(tmbId)) {
+ setSelected((state) => state.filter((tmb) => tmb.tmbId !== tmbId));
+ // setSelectedTmbIds((state) => state.filter((tmbId) => tmbId !== memberId));
} else {
- setSelectedMembers((state) => [...state, memberId]);
+ // setSelectedTmbIds((state) => [...state, memberId]);
+ const member = allMembers.find((item) => item.tmbId === tmbId)!;
+ setSelected((state) => [
+ ...state,
+ {
+ name: member.memberName,
+ tmbId,
+ avatar: member.avatar
+ }
+ ]);
}
};
@@ -123,7 +151,14 @@ function OrgMemberManageModal({
gridTemplateColumns="1fr 1fr"
h={'100%'}
>
-
+
-
+
{filterMembers.map((member) => {
return (
{/* */}
-
- {`${t('common:chosen')}:${selectedMembers.length}`}
- {selectedMembers.map((tmbId) => {
- const member = allMembers.find((item) => item.tmbId === tmbId)!;
+
+ {`${t('common:chosen')}:${selected.length}`}
+ {selected.map((member) => {
return (
- {member?.memberName}
+ {member?.name}
handleToggleSelect(tmbId)}
+ onClick={() => handleToggleSelect(member.tmbId)}
/>
);
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
index 96e2e705a..ad005413c 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMoveModal.tsx
@@ -1,4 +1,4 @@
-import { putMoveOrg } from '@/web/support/user/team/org/api';
+import { getOrgList, putMoveOrg } from '@/web/support/user/team/org/api';
import { Button, ModalBody, ModalFooter } from '@chakra-ui/react';
import type { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import MyModal from '@fastgpt/web/components/common/MyModal';
@@ -6,17 +6,15 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import { useMemo, useState } from 'react';
import OrgTree from './OrgTree';
-import dynamic from 'next/dynamic';
import { useUserStore } from '@/web/support/user/useUserStore';
+import useOrg from '@/web/support/user/team/org/hooks/useOrg';
function OrgMoveModal({
movingOrg,
- orgs,
onClose,
onSuccess
}: {
movingOrg: OrgListItemType;
- orgs: OrgListItemType[];
onClose: () => void;
onSuccess: () => void;
}) {
@@ -32,11 +30,6 @@ function OrgMoveModal({
}
});
- const filterMovingOrgs = useMemo(
- () => orgs.filter((org) => org._id !== movingOrg._id),
- [movingOrg._id, orgs]
- );
-
return (
-
+
{isExpanded &&
- children.length > 0 &&
- children.map((child) => (
+ orgs.length > 0 &&
+ orgs.map((child) => (
@@ -84,19 +90,29 @@ function OrgTreeNode({
}
function OrgTree({
- orgs,
selectedOrg,
setSelectedOrg
}: {
- orgs: OrgType[];
- selectedOrg?: OrgType;
- setSelectedOrg: (org?: OrgType) => void;
+ selectedOrg?: OrgListItemType;
+ setSelectedOrg: (org?: OrgListItemType) => void;
}) {
- const root = orgs[0];
- if (!root) return;
+ const { userInfo } = useUserStore();
+ const root: OrgListItemType = {
+ _id: '',
+ path: '',
+ pathId: '',
+ name: userInfo?.team.teamName || '',
+ avatar: userInfo?.team.avatar || ''
+ } as any;
return (
-
+
);
}
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
index 4c831164d..1249bde30 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
@@ -24,12 +24,7 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import { useMemo, useState } from 'react';
import MemberTag from '@/components/support/user/team/Info/MemberTag';
-import {
- deleteOrg,
- deleteOrgMember,
- getOrgList,
- getOrgMembers
-} from '@/web/support/user/team/org/api';
+import { deleteOrg, deleteOrgMember } from '@/web/support/user/team/org/api';
import IconButton from './IconButton';
import { defaultOrgForm, type OrgFormType } from './OrgInfoModal';
@@ -43,6 +38,7 @@ import { useSystemStore } from '@/web/common/system/useSystemStore';
import { delRemoveMember } from '@/web/support/user/team/api';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import useOrg from '@/web/support/user/team/org/hooks/useOrg';
const OrgInfoModal = dynamic(() => import('./OrgInfoModal'));
const OrgMemberManageModal = dynamic(() => import('./OrgMemberManageModal'));
@@ -82,51 +78,25 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
const { userInfo, isTeamAdmin } = useUserStore();
const { feConfigs } = useSystemStore();
const isSyncMember = feConfigs.register_method?.includes('sync');
-
- const [searchOrg, setSearchOrg] = useState('');
-
- const [parentId, setParentId] = useState();
- const [currentOrg, setCurrentOrg] = useState();
-
- // 用于 org 层级
- const [orgStack, setOrgStack] = useState([]);
-
- const {
- data: orgs = [],
- loading: isLoadingOrgs,
- refresh: refetchOrgs
- } = useRequest2(
- () => {
- return getOrgList(parentId);
- },
- {
- manual: false,
- refreshDeps: [userInfo?.team?.teamId, parentId]
- }
- );
-
- const paths = useMemo(() => {
- if (!currentOrg) return [];
- return orgStack
- .map((org) => {
- return {
- parentId: getOrgChildrenPath(org),
- parentName: org.name
- };
- })
- .filter(Boolean) as ParentTreePathItemType[];
- }, [currentOrg, orgStack]);
-
- const onClickOrg = (org: OrgListItemType) => {
- setParentId(currentOrg?._id);
- setOrgStack([...orgStack, org]);
- setCurrentOrg(org);
- };
-
const [editOrg, setEditOrg] = useState();
const [manageMemberOrg, setManageMemberOrg] = useState();
const [movingOrg, setMovingOrg] = useState();
+ const [searchOrg, setSearchOrg] = useState('');
+
+ const {
+ currentOrg,
+ orgs,
+ isLoadingOrgs,
+ paths,
+ onClickOrg,
+ members,
+ MemberScrollData,
+ onPathClick,
+ refresh,
+ updateCurrentOrg
+ } = useOrg();
+
// Delete org
const { ConfirmModal: ConfirmDeleteOrgModal, openConfirm: openDeleteOrgModal } = useConfirm({
type: 'delete',
@@ -134,17 +104,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
});
const deleteOrgHandler = (orgId: string) => openDeleteOrgModal(() => deleteOrgReq(orgId))();
const { runAsync: deleteOrgReq } = useRequest2(deleteOrg, {
- onSuccess: () => {
- refetchOrgs();
- }
- });
-
- const { data: members = [], ScrollData: MemberScrollData } = useScrollPagination(getOrgMembers, {
- pageSize: 20,
- params: {
- orgId: currentOrg?._id
- },
- refreshDeps: [currentOrg?._id]
+ onSuccess: refresh
});
// Delete member
@@ -159,15 +119,11 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
});
const { runAsync: deleteMemberReq } = useRequest2(deleteOrgMember, {
- onSuccess: () => {
- refetchOrgs();
- }
+ onSuccess: refresh
});
const { runAsync: deleteMemberFromTeamReq } = useRequest2(delRemoveMember, {
- onSuccess: () => {
- refetchOrgs();
- }
+ onSuccess: refresh
});
const searchedOrgs = useMemo(() => {
@@ -196,7 +152,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
isLoading={isLoadingOrgs}
>
-
+
@@ -356,8 +312,14 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
label: t('account_team:delete_from_org'),
onClick: () =>
openDeleteMemberFromOrgModal(
- () =>
- deleteMemberReq(currentOrg._id, member.tmbId),
+ () => {
+ if (currentOrg) {
+ return deleteMemberReq(
+ currentOrg._id,
+ member.tmbId
+ );
+ }
+ },
undefined,
t('account_team:confirm_delete_from_org', {
username: member.memberName
@@ -383,20 +345,15 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
{!isSyncMember && (
-
+
- {currentOrg?.name || userInfo?.team.teamName}
+ {currentOrg.name}
- {currentOrg && currentOrg?.path !== '' && (
+ {currentOrg?.path !== '' && (
setEditOrg(currentOrg)} />
)}
- {currentOrg && (
+ {currentOrg?.path !== '' && (
{currentOrg?.description || t('common:common.no_intro')}
)}
@@ -413,7 +370,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
onClick={() => {
setEditOrg({
...defaultOrgForm,
- parentId: currentOrg?._id
+ path: currentOrg.path
});
}}
/>
@@ -422,7 +379,7 @@ function OrgTable({ Tabs }: { Tabs: React.ReactNode }) {
text={t('account_team:manage_member')}
onClick={() => setManageMemberOrg(currentOrg)}
/>
- {currentOrg && currentOrg?.path !== '' && (
+ {currentOrg?.path !== '' && (
<>
setEditOrg(undefined)}
- onSuccess={refetchOrgs}
+ onSuccess={refresh}
+ updateCurrentOrg={updateCurrentOrg}
/>
)}
{!!movingOrg && (
setMovingOrg(undefined)}
- onSuccess={refetchOrgs}
+ onSuccess={refresh}
/>
)}
{!!manageMemberOrg && (
setManageMemberOrg(undefined)}
/>
)}
diff --git a/projects/app/src/pageComponents/account/team/context.tsx b/projects/app/src/pageComponents/account/team/context.tsx
index b1aa6730b..0c29585b6 100644
--- a/projects/app/src/pageComponents/account/team/context.tsx
+++ b/projects/app/src/pageComponents/account/team/context.tsx
@@ -23,21 +23,18 @@ const EditInfoModal = dynamic(() => import('./EditInfoModal'));
type TeamModalContextType = {
myTeams: TeamTmbItemType[];
members: TeamMemberItemType[];
- groups: MemberGroupListType;
isLoading: boolean;
onSwitchTeam: (teamId: string) => void;
setEditTeamData: React.Dispatch>;
refetchMembers: () => void;
refetchTeams: () => void;
- refetchGroups: () => void;
teamSize: number;
MemberScrollData: ReturnType['ScrollData'];
};
export const TeamContext = createContext({
myTeams: [],
- groups: [],
members: [],
isLoading: false,
onSwitchTeam: function (_teamId: string): void {
@@ -52,9 +49,6 @@ export const TeamContext = createContext({
refetchMembers: function (): void {
throw new Error('Function not implemented.');
},
- refetchGroups: function (): void {
- throw new Error('Function not implemented.');
- },
teamSize: 0,
MemberScrollData: () => <>>
});
@@ -110,16 +104,7 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
}
);
- const {
- data: groups = [],
- loading: isLoadingGroups,
- refresh: refetchGroups
- } = useRequest2(getGroupList, {
- manual: false,
- refreshDeps: [userInfo?.team?.teamId]
- });
-
- const isLoading = isLoadingTeams || isSwitchingTeam || loadingMembers || isLoadingGroups;
+ const isLoading = isLoadingTeams || isSwitchingTeam || loadingMembers;
const contextValue = {
myTeams,
@@ -131,8 +116,6 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
setEditTeamData,
members,
refetchMembers,
- groups,
- refetchGroups,
teamSize: teamMemberCountData?.count || 0,
MemberScrollData
};
diff --git a/projects/app/src/web/support/user/team/org/api.ts b/projects/app/src/web/support/user/team/org/api.ts
index 825ec5446..b2b743610 100644
--- a/projects/app/src/web/support/user/team/org/api.ts
+++ b/projects/app/src/web/support/user/team/org/api.ts
@@ -10,8 +10,8 @@ import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/t
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
-export const getOrgList = (parentId: ParentIdType) =>
- GET(`/proApi/support/user/team/org/list`, { parentId });
+export const getOrgList = (params: { orgPath: string; getPermission?: boolean }) =>
+ GET(`/proApi/support/user/team/org/list`, params);
export const postCreateOrg = (data: postCreateOrgData) =>
POST('/proApi/support/user/team/org/create', data);
@@ -19,19 +19,17 @@ export const postCreateOrg = (data: postCreateOrgData) =>
export const deleteOrg = (orgId: string) =>
DELETE('/proApi/support/user/team/org/delete', { orgId });
-export const deleteOrgMember = (orgId: string, tmbId: string) =>
- DELETE('/proApi/support/user/team/org/deleteMember', { orgId, tmbId });
-
export const putMoveOrg = (data: putMoveOrgType) => PUT('/proApi/support/user/team/org/move', data);
export const putUpdateOrg = (data: putUpdateOrgData) =>
PUT('/proApi/support/user/team/org/update', data);
+// org members
export const putUpdateOrgMembers = (data: putUpdateOrgMembersData) =>
PUT('/proApi/support/user/team/org/updateMembers', data);
-// export const putChnageOrgOwner = (data: putChnageOrgOwnerData) =>
-// PUT('/proApi/support/user/team/org/changeOwner', data);
-
-export const getOrgMembers = (data: PaginationProps<{ orgId: string }>) =>
+export const getOrgMembers = (data: PaginationProps<{ orgPath?: string }>) =>
GET>(`/proApi/support/user/team/org/members`, data);
+
+export const deleteOrgMember = (orgId: string, tmbId: string) =>
+ DELETE('/proApi/support/user/team/org/deleteMember', { orgId, tmbId });
diff --git a/projects/app/src/web/support/user/team/org/hooks/useOrg.tsx b/projects/app/src/web/support/user/team/org/hooks/useOrg.tsx
new file mode 100644
index 000000000..d5db3805e
--- /dev/null
+++ b/projects/app/src/web/support/user/team/org/hooks/useOrg.tsx
@@ -0,0 +1,108 @@
+import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
+import { OrgListItemType } from '@fastgpt/global/support/user/team/org/type';
+import { memo, useMemo, useState } from 'react';
+import { useUserStore } from '../../../useUserStore';
+import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import { getOrgList, getOrgMembers } from '../api';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+
+function useOrg({ getPermission = true }: { getPermission?: boolean } = {}) {
+ const [orgStack, setOrgStack] = useState([]);
+
+ const { userInfo } = useUserStore();
+
+ const path = useMemo(
+ () => (orgStack.length ? getOrgChildrenPath(orgStack[orgStack.length - 1]) : ''),
+ [orgStack]
+ );
+
+ const currentOrg = useMemo(() => {
+ return (
+ orgStack.at(-1) ??
+ ({
+ _id: '',
+ path: '',
+ pathId: '',
+ avatar: userInfo?.team.avatar,
+ name: userInfo?.team.teamName
+ } as OrgListItemType) // root org
+ );
+ }, [orgStack, userInfo?.team.avatar, userInfo?.team.teamName]);
+
+ const {
+ data: orgs = [],
+ loading: isLoadingOrgs,
+ refresh: refetchOrgs
+ } = useRequest2(() => getOrgList({ orgPath: path, getPermission }), {
+ manual: false,
+ refreshDeps: [userInfo?.team?.teamId, path]
+ });
+
+ const paths = useMemo(() => {
+ if (!currentOrg) return [];
+ return orgStack
+ .map((org) => {
+ return {
+ parentId: getOrgChildrenPath(org),
+ parentName: org.name
+ };
+ })
+ .filter(Boolean) as ParentTreePathItemType[];
+ }, [currentOrg, orgStack]);
+
+ const onClickOrg = (org: OrgListItemType) => {
+ setOrgStack([...orgStack, org]);
+ };
+
+ const {
+ data: members = [],
+ ScrollData: MemberScrollData,
+ refreshList: refetchMembers
+ } = useScrollPagination(getOrgMembers, {
+ pageSize: 20,
+ params: {
+ orgPath: path
+ },
+ refreshDeps: [path]
+ });
+
+ const onPathClick = (path: string) => {
+ const pathIds = path.split('/');
+ setOrgStack(orgStack.filter((org) => pathIds.includes(org.pathId)));
+ };
+
+ const refresh = () => {
+ refetchOrgs();
+ refetchMembers();
+ };
+
+ const updateCurrentOrg = (data: { name?: string; description?: string; avatar?: string }) => {
+ if (currentOrg.path === '') return;
+ setOrgStack([
+ ...orgStack.slice(0, -1),
+ {
+ ...currentOrg,
+ name: data.name || currentOrg.name,
+ description: data.description || currentOrg.description,
+ avatar: data.avatar || currentOrg.avatar
+ }
+ ]);
+ };
+
+ return {
+ orgStack,
+ currentOrg,
+ orgs,
+ isLoadingOrgs,
+ paths,
+ onClickOrg,
+ members,
+ MemberScrollData,
+ onPathClick,
+ refresh,
+ updateCurrentOrg
+ };
+}
+
+export default useOrg;
From 64fb09146f9a208b35e850ff5b0da794172ed387 Mon Sep 17 00:00:00 2001
From: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Date: Tue, 25 Mar 2025 22:31:31 +0800
Subject: [PATCH 22/35] fix: member list, login button (#4322)
---
.../support/permission/MemberManager/MemberModal.tsx | 2 --
.../account/team/OrgManage/OrgMemberManageModal.tsx | 5 +----
.../app/src/pageComponents/login/LoginForm/FormLayout.tsx | 1 +
3 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
index 22e56d470..42534f59c 100644
--- a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
+++ b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
@@ -66,8 +66,6 @@ function MemberModal({
members: orgMembers,
MemberScrollData: OrgMemberScrollData,
onPathClick,
- refresh,
- updateCurrentOrg,
orgs
} = useOrg({ getPermission: false });
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
index aec30c1b0..0bb789c86 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgMemberManageModal.tsx
@@ -56,10 +56,7 @@ function OrgMemberManageModal({
ScrollData: MemberScrollData,
isLoading: isLoadingMembers
} = useScrollPagination(getTeamMembers, {
- pageSize: 20,
- params: {
- withLeaved: false
- }
+ pageSize: 20
});
const {
diff --git a/projects/app/src/pageComponents/login/LoginForm/FormLayout.tsx b/projects/app/src/pageComponents/login/LoginForm/FormLayout.tsx
index 286cd5cb1..db310bc06 100644
--- a/projects/app/src/pageComponents/login/LoginForm/FormLayout.tsx
+++ b/projects/app/src/pageComponents/login/LoginForm/FormLayout.tsx
@@ -134,6 +134,7 @@ const FormLayout = ({ children, setPageType, pageType }: Props) => {
});
router.replace(item.redirectUrl, '_self');
}
+ item.pageType && setPageType(item.pageType);
},
[lastRoute, router, setLoginStore, setPageType]
);
From 4871a6980f395e12e897bd8f0d8e0390f86cedf6 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Wed, 26 Mar 2025 00:02:14 +0800
Subject: [PATCH 23/35] perf: member group (#4324)
* sync collection
* remove lock
* perf: member group
---
.../support/permission/memberGroup/api.d.ts | 4 +
.../support/permission/memberGroup/type.d.ts | 33 ++--
packages/global/support/user/api.d.ts | 5 +-
.../service/support/user/team/controller.ts | 8 +
.../permission/MemberManager/MemberModal.tsx | 11 +-
.../support/user/team/Info/MemberTag.tsx | 8 +-
.../team/GroupManage/GroupInfoModal.tsx | 42 ++---
.../team/GroupManage/GroupManageMember.tsx | 17 +-
.../GroupManage/GroupTransferOwnerModal.tsx | 47 +++---
.../account/team/GroupManage/index.tsx | 120 ++++++--------
.../account/team/MemberTable.tsx | 149 ++++++++----------
.../account/team/OrgManage/index.tsx | 5 +-
.../pageComponents/account/team/context.tsx | 10 +-
.../src/web/support/user/team/group/api.ts | 9 +-
14 files changed, 213 insertions(+), 255 deletions(-)
create mode 100644 packages/global/support/permission/memberGroup/api.d.ts
diff --git a/packages/global/support/permission/memberGroup/api.d.ts b/packages/global/support/permission/memberGroup/api.d.ts
new file mode 100644
index 000000000..0e81cc1d4
--- /dev/null
+++ b/packages/global/support/permission/memberGroup/api.d.ts
@@ -0,0 +1,4 @@
+export type GetGroupListBody = {
+ searchKey?: string;
+ withMembers?: boolean;
+};
diff --git a/packages/global/support/permission/memberGroup/type.d.ts b/packages/global/support/permission/memberGroup/type.d.ts
index f70e9a89e..5011d74f6 100644
--- a/packages/global/support/permission/memberGroup/type.d.ts
+++ b/packages/global/support/permission/memberGroup/type.d.ts
@@ -1,6 +1,7 @@
import { TeamMemberItemType } from 'support/user/team/type';
import { TeamPermission } from '../user/controller';
import { GroupMemberRole } from './constant';
+import { Permission } from '../controller';
type MemberGroupSchemaType = {
_id: string;
@@ -16,23 +17,25 @@ type GroupMemberSchemaType = {
role: `${GroupMemberRole}`;
};
-type MemberGroupType = MemberGroupSchemaType & {
- members: {
- tmbId: string;
- name: string;
- avatar: string;
- }[];
- count: number;
- owner: {
- tmbId: string;
- name: string;
- avatar: string;
- };
- canEdit: boolean;
+type MemberGroupListItemType = MemberGroupSchemaType & {
+ members: T extends true
+ ? {
+ tmbId: string;
+ name: string;
+ avatar: string;
+ }[]
+ : undefined;
+ count: T extends true ? number : undefined;
+ owner?: T extends true
+ ? {
+ tmbId: string;
+ name: string;
+ avatar: string;
+ }
+ : undefined;
+ permission: T extends true ? Permission : undefined;
};
-type MemberGroupListType = MemberGroupType[];
-
type GroupMemberItemType = {
tmbId: string;
name: string;
diff --git a/packages/global/support/user/api.d.ts b/packages/global/support/user/api.d.ts
index 2f8b4f5bd..8c5036d4a 100644
--- a/packages/global/support/user/api.d.ts
+++ b/packages/global/support/user/api.d.ts
@@ -1,4 +1,7 @@
-import { MemberGroupSchemaType, MemberGroupType } from 'support/permission/memberGroup/type';
+import {
+ MemberGroupSchemaType,
+ MemberGroupListItemType
+} from 'support/permission/memberGroup/type';
import { OAuthEnum } from './constant';
import { TrackRegisterParams } from './login/api';
import { TeamMemberStatusEnum } from './team/constant';
diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts
index b90dff737..8c3c4cf10 100644
--- a/packages/service/support/user/team/controller.ts
+++ b/packages/service/support/user/team/controller.ts
@@ -55,6 +55,14 @@ async function getTeamMember(match: Record): Promise {
+ const tmb = await MongoTeamMember.findOne({
+ teamId,
+ role: TeamMemberRoleEnum.owner
+ }).lean();
+ return tmb;
+};
+
export async function getTmbInfoByTmbId({ tmbId }: { tmbId: string }) {
if (!tmbId) {
return Promise.reject('tmbId or userId is required');
diff --git a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
index 42534f59c..02b762893 100644
--- a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
+++ b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
@@ -28,20 +28,17 @@ import {
DEFAULT_USER_AVATAR
} from '@fastgpt/global/common/system/constants';
import Path from '@/components/common/folder/Path';
-import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
-import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
import { useContextSelector } from 'use-context-selector';
import { CollaboratorContext } from './context';
import { getTeamMembers } from '@/web/support/user/team/api';
import { getGroupList } from '@/web/support/user/team/group/api';
-import { getOrgList, getOrgMembers } from '@/web/support/user/team/org/api';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import MemberItemCard from './MemberItemCard';
import { GetSearchUserGroupOrg } from '@/web/support/user/api';
import useOrg from '@/web/support/user/team/org/hooks/useOrg';
-import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
+import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
const HoverBoxStyle = {
bgColor: 'myGray.50',
@@ -76,7 +73,9 @@ function MemberModal({
const { data: groups = [], loading: loadingGroupsAndOrgs } = useRequest2(
async () => {
if (!userInfo?.team?.teamId) return [];
- return getGroupList();
+ return getGroupList({
+ withMembers: false
+ });
},
{
manual: false,
@@ -117,7 +116,7 @@ function MemberModal({
return members;
}, [searchText, members, searchedData?.members]);
- const [selectedGroupList, setSelectedGroupList] = useState([]);
+ const [selectedGroupList, setSelectedGroupList] = useState[]>([]);
const filterGroups = useMemo(() => {
if (searchText) {
return searchedData?.groups.map((item) => ({
diff --git a/projects/app/src/components/support/user/team/Info/MemberTag.tsx b/projects/app/src/components/support/user/team/Info/MemberTag.tsx
index c62f0f1cc..3868f37d4 100644
--- a/projects/app/src/components/support/user/team/Info/MemberTag.tsx
+++ b/projects/app/src/components/support/user/team/Info/MemberTag.tsx
@@ -3,16 +3,16 @@ import Avatar from '@fastgpt/web/components/common/Avatar';
import React from 'react';
type Props = {
- name: string;
- avatar: string;
+ name?: string;
+ avatar?: string;
};
function MemberTag({ name, avatar }: Props) {
return (
-
+ {avatar && }
- {name}
+ {name || '-'}
);
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
index c86134bfe..d97b87590 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
@@ -4,15 +4,13 @@ import Avatar from '@fastgpt/web/components/common/Avatar';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import { useTranslation } from 'next-i18next';
-import React, { useMemo } from 'react';
+import React from 'react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useForm } from 'react-hook-form';
-import { useContextSelector } from 'use-context-selector';
-import { TeamContext } from '../context';
import { postCreateGroup, putUpdateGroup } from '@/web/support/user/team/group/api';
import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
-import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
export type GroupFormType = {
avatar: string;
@@ -21,17 +19,15 @@ export type GroupFormType = {
function GroupInfoModal({
onClose,
- editGroupId,
- groups,
- refetchGroups
+ editGroup,
+ onSuccess
}: {
onClose: () => void;
- editGroupId?: string;
- groups: MemberGroupListType;
- refetchGroups: () => void;
+ editGroup?: MemberGroupListItemType;
+ onSuccess: () => void;
}) {
- const { refetchMembers } = useContextSelector(TeamContext, (v) => v);
const { t } = useTranslation();
+
const {
File: AvatarSelect,
onOpen: onOpenSelectAvatar,
@@ -41,14 +37,10 @@ function GroupInfoModal({
multiple: false
});
- const group = useMemo(() => {
- return groups.find((item) => item._id === editGroupId);
- }, [editGroupId, groups]);
-
const { register, handleSubmit, getValues, setValue } = useForm({
defaultValues: {
- name: group?.name || '',
- avatar: group?.avatar || DEFAULT_TEAM_AVATAR
+ name: editGroup?.name || '',
+ avatar: editGroup?.avatar || DEFAULT_TEAM_AVATAR
}
});
@@ -74,21 +66,21 @@ function GroupInfoModal({
});
},
{
- onSuccess: () => Promise.all([onClose(), refetchGroups(), refetchMembers()])
+ onSuccess: () => Promise.all([onClose(), onSuccess()])
}
);
const { runAsync: onUpdate, loading: isLoadingUpdate } = useRequest2(
async (data: GroupFormType) => {
- if (!editGroupId) return;
+ if (!editGroup) return;
return putUpdateGroup({
- groupId: editGroupId,
+ groupId: editGroup._id,
name: data.name,
avatar: data.avatar
});
},
{
- onSuccess: () => Promise.all([onClose(), refetchGroups(), refetchMembers()])
+ onSuccess: () => Promise.all([onClose(), onSuccess()])
}
);
@@ -97,8 +89,8 @@ function GroupInfoModal({
return (
{t('user:team.avatar_and_name')}
@@ -120,14 +112,14 @@ function GroupInfoModal({
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index 7b13fc301..24341ba32 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -26,7 +26,7 @@ import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import {
GroupMemberItemType,
- MemberGroupListType
+ MemberGroupListItemType
} from '@fastgpt/global/support/permission/memberGroup/type';
import { useMount } from 'ahooks';
@@ -43,24 +43,19 @@ export type GroupFormType = {
function GroupEditModal({
onClose,
editGroupId,
- groups,
- refetchGroups
+ group,
+ onSuccess
}: {
onClose: () => void;
editGroupId?: string;
- groups: MemberGroupListType;
- refetchGroups: () => void;
+ group: MemberGroupListItemType;
+ onSuccess: () => void;
}) {
const { t } = useTranslation();
const { userInfo } = useUserStore();
const { toast } = useToast();
- const group = useMemo(() => {
- return groups.find((item) => item._id === editGroupId);
- }, [editGroupId, groups]);
-
const allMembers = useContextSelector(TeamContext, (v) => v.members);
- const refetchMembers = useContextSelector(TeamContext, (v) => v.refetchMembers);
const MemberScrollData = useContextSelector(TeamContext, (v) => v.MemberScrollData);
const [hoveredMemberId, setHoveredMemberId] = useState();
@@ -94,7 +89,7 @@ function GroupEditModal({
});
},
{
- onSuccess: () => Promise.all([onClose(), refetchGroups(), refetchMembers()])
+ onSuccess: () => Promise.all([onClose(), onSuccess()])
}
);
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
index 93b0bc7cc..48741fc91 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
@@ -1,4 +1,4 @@
-import { putGroupChangeOwner, putUpdateGroup } from '@/web/support/user/team/group/api';
+import { putGroupChangeOwner } from '@/web/support/user/team/group/api';
import {
Box,
Flex,
@@ -15,26 +15,24 @@ import Avatar from '@fastgpt/web/components/common/Avatar';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import React, { useMemo, useState } from 'react';
+import React, { useState } from 'react';
import { TeamContext } from '../context';
import { useContextSelector } from 'use-context-selector';
-import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
import { GetSearchUserGroupOrg } from '@/web/support/user/api';
import { Omit } from '@fastgpt/web/components/common/DndDrag';
-export type ChangeOwnerModalProps = {
- groupId: string;
- groups: MemberGroupListType;
- refetchGroups: () => void;
-};
-
export function ChangeOwnerModal({
- onClose,
- groupId,
- groups,
- refetchGroups
-}: ChangeOwnerModalProps & { onClose: () => void }) {
+ group,
+ onSuccess,
+ onClose
+}: {
+ group: MemberGroupListItemType;
+ onSuccess: () => void;
+ onClose: () => void;
+}) {
const { t } = useTranslation();
+
const [inputValue, setInputValue] = React.useState('');
const { data: searchedData } = useRequest2(
async () => {
@@ -50,14 +48,8 @@ export function ChangeOwnerModal({
);
const { members: allMembers } = useContextSelector(TeamContext, (v) => v);
- const group = useMemo(() => {
- return groups.find((item) => item._id === groupId);
- }, [groupId, groups]);
-
const memberList = searchedData ? searchedData.members : allMembers;
- const [keepAdmin, setKeepAdmin] = useState(true);
-
const {
isOpen: isOpenMemberListMenu,
onClose: onCloseMemberListMenu,
@@ -69,10 +61,12 @@ export function ChangeOwnerModal({
'permission' | 'teamId'
> | null>(null);
- const { runAsync, loading } = useRequest2(
- (tmbId: string) => putGroupChangeOwner(groupId, tmbId),
+ const [keepAdmin, setKeepAdmin] = useState(true);
+
+ const { runAsync: onTransfer, loading } = useRequest2(
+ (tmbId: string) => putGroupChangeOwner(group._id, tmbId),
{
- onSuccess: () => Promise.all([onClose(), refetchGroups()]),
+ onSuccess: () => Promise.all([onClose(), onSuccess()]),
successToast: t('common:permission.change_owner_success'),
errorToast: t('common:permission.change_owner_failed')
}
@@ -82,7 +76,7 @@ export function ChangeOwnerModal({
if (!selectedMember) {
return;
}
- await runAsync(selectedMember.tmbId);
+ await onTransfer(selectedMember.tmbId);
};
return (
@@ -92,7 +86,6 @@ export function ChangeOwnerModal({
iconColor="primary.600"
onClose={onClose}
title={t('common:permission.change_owner')}
- isLoading={loading}
>
@@ -181,7 +174,9 @@ export function ChangeOwnerModal({
-
+
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
index 574e52117..78bb0d60c 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/index.tsx
@@ -3,7 +3,6 @@ import {
Box,
Button,
Flex,
- HStack,
Table,
TableContainer,
Tbody,
@@ -16,20 +15,18 @@ import {
import { useTranslation } from 'next-i18next';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import MyBox from '@fastgpt/web/components/common/MyBox';
-import { useContextSelector } from 'use-context-selector';
-import { TeamContext } from '../context';
import MyMenu, { MenuItemType } from '@fastgpt/web/components/common/MyMenu';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { deleteGroup, getGroupList, getGroupMembers } from '@/web/support/user/team/group/api';
+import { deleteGroup, getGroupList } from '@/web/support/user/team/group/api';
import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant';
import MemberTag from '../../../../components/support/user/team/Info/MemberTag';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import dynamic from 'next/dynamic';
import { useState } from 'react';
import IconButton from '../OrgManage/IconButton';
-import { MemberGroupType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
const ChangeOwnerModal = dynamic(() => import('./GroupTransferOwnerModal'));
const GroupInfoModal = dynamic(() => import('./GroupInfoModal'));
@@ -39,26 +36,23 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { userInfo } = useUserStore();
- const { members, teamSize } = useContextSelector(TeamContext, (v) => v);
-
const {
data: groups = [],
loading: isLoadingGroups,
refresh: refetchGroups
- } = useRequest2(getGroupList, {
+ } = useRequest2(() => getGroupList({ withMembers: true }), {
manual: false,
refreshDeps: [userInfo?.team?.teamId]
});
- const [editGroup, setEditGroup] = useState();
+ const [editGroup, setEditGroup] = useState>();
const {
isOpen: isOpenGroupInfo,
onOpen: onOpenGroupInfo,
onClose: onCloseGroupInfo
} = useDisclosure();
-
- const onEditGroupInfo = (e: MemberGroupType) => {
+ const onEditGroupInfo = (e: MemberGroupListItemType) => {
setEditGroup(e);
onOpenGroupInfo();
};
@@ -67,7 +61,6 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
type: 'delete',
content: t('account_team:confirm_delete_group')
});
-
const { runAsync: delDeleteGroup } = useRequest2(deleteGroup, {
onSuccess: () => {
refetchGroups();
@@ -79,21 +72,17 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
onOpen: onOpenManageGroupMember,
onClose: onCloseManageGroupMember
} = useDisclosure();
- const onManageMember = (e: MemberGroupType) => {
+ const onManageMember = (e: MemberGroupListItemType) => {
setEditGroup(e);
onOpenManageGroupMember();
};
- const hasGroupManagePer = (group: (typeof groups)[0]) => userInfo?.team.permission.hasManagePer;
-
- const isGroupOwner = (group: (typeof groups)[0]) => userInfo?.team.permission.hasManagePer;
-
const {
isOpen: isOpenChangeOwner,
onOpen: onOpenChangeOwner,
onClose: onCloseChangeOwner
} = useDisclosure();
- const onChangeOwner = (e: MemberGroupType) => {
+ const onChangeOwner = (e: MemberGroupListItemType) => {
setEditGroup(e);
onOpenChangeOwner();
};
@@ -134,58 +123,38 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{groups?.map((group) => (
- |
-
-
- ({group.name === DefaultGroupName ? teamSize : group.count})
-
- |
item.role === 'owner')?.memberName ?? ''
- : group.owner.name
- }
- avatar={
- group.name === DefaultGroupName
- ? members.find((item) => item.role === 'owner')?.avatar ?? ''
- : group.owner.avatar
+ group.name === DefaultGroupName ? userInfo?.team.teamName ?? '' : group.name
}
+ avatar={group.avatar}
/>
|
- {group.name === DefaultGroupName ? (
- v.avatar)} total={teamSize} />
- ) : hasGroupManagePer(group) ? (
-
- onManageMember(group)}>
- members.find((m) => m.tmbId === v.tmbId)?.avatar ?? ''
- )}
- total={group.count}
- />
-
-
- ) : (
- members.find((m) => m.tmbId === v.tmbId)?.avatar ?? ''
- )}
- total={group.count}
- />
- )}
+
|
- {hasGroupManagePer(group) && group.name !== DefaultGroupName && (
+
+ onManageMember(group)
+ }
+ : {})}
+ >
+ v.avatar)}
+ total={group.count}
+ />
+
+
+ |
+
+ {group.permission?.hasManagePer && (
}
menuList={[
@@ -205,7 +174,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
onManageMember(group);
}
},
- ...(isGroupOwner(group)
+ ...(group.permission?.isOwner
? [
{
label: t('account_team:transfer_ownership'),
@@ -239,34 +208,33 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
- {isOpenChangeOwner && editGroup && (
-
- )}
+
{isOpenGroupInfo && (
{
onCloseGroupInfo();
setEditGroup(undefined);
}}
- editGroupId={editGroup?._id}
/>
)}
+ {isOpenChangeOwner && editGroup && (
+
+ )}
+
{isOpenManageGroupMember && editGroup && (
{
onCloseManageGroupMember();
setEditGroup(undefined);
}}
- editGroupId={editGroup._id}
+ onSuccess={refetchGroups}
/>
)}
>
diff --git a/projects/app/src/pageComponents/account/team/MemberTable.tsx b/projects/app/src/pageComponents/account/team/MemberTable.tsx
index edc1eedf4..cb8c5e4ea 100644
--- a/projects/app/src/pageComponents/account/team/MemberTable.tsx
+++ b/projects/app/src/pageComponents/account/team/MemberTable.tsx
@@ -20,8 +20,9 @@ import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useEditTextarea } from '@fastgpt/web/hooks/useEditTextarea';
import {
delRemoveMember,
- postRestoreMember,
- putUpdateMemberNameByManager
+ getTeamMembers,
+ putUpdateMemberNameByManager,
+ postRestoreMember
} from '@/web/support/user/team/api';
import Tag from '@fastgpt/web/components/common/Tag';
import Icon from '@fastgpt/web/components/common/Icon';
@@ -33,7 +34,6 @@ import dynamic from 'next/dynamic';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { delLeaveTeam } from '@/web/support/user/team/api';
import { GetSearchUserGroupOrg, postSyncMembers } from '@/web/support/user/api';
-import MyLoading from '@fastgpt/web/components/common/MyLoading';
import {
TeamMemberRoleEnum,
TeamMemberStatusEnum
@@ -41,10 +41,12 @@ import {
import format from 'date-fns/format';
import OrgTags from '@/components/support/user/team/OrgTags';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
-import { useState } from 'react';
+import { useCallback, useState } from 'react';
import { downloadFetch } from '@/web/common/system/utils';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
import { useToast } from '@fastgpt/web/hooks/useToast';
+import MyBox from '@fastgpt/web/components/common/MyBox';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
const InviteModal = dynamic(() => import('./Invite/InviteModal'));
const TeamTagModal = dynamic(() => import('@/components/support/user/team/TeamTagModal'));
@@ -55,20 +57,72 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { userInfo } = useUserStore();
const { feConfigs } = useSystemStore();
+ const isSyncMember = feConfigs?.register_method?.includes('sync');
- const { myTeams, refetchTeams, members, refetchMembers, onSwitchTeam, MemberScrollData } =
- useContextSelector(TeamContext, (v) => v);
+ const { myTeams, onSwitchTeam } = useContextSelector(TeamContext, (v) => v);
const {
isOpen: isOpenTeamTagsAsync,
onOpen: onOpenTeamTagsAsync,
onClose: onCloseTeamTagsAsync
} = useDisclosure();
+
+ // member action
+ const {
+ data: members = [],
+ isLoading: loadingMembers,
+ refreshList: refetchMemberList,
+ ScrollData: MemberScrollData
+ } = useScrollPagination(getTeamMembers, {
+ pageSize: 20,
+ params: {
+ withLeaved: true
+ }
+ });
+
+ const [searchText, setSearchText] = useState('');
+ const { data: searchMembersData, run: refreshSearchMembers } = useRequest2(
+ async () => {
+ if (!searchText) return Promise.resolve();
+ return GetSearchUserGroupOrg(searchText, { members: true, orgs: false, groups: false });
+ },
+ {
+ manual: false,
+ throttleWait: 500,
+ refreshDeps: [searchText]
+ }
+ );
+
+ const onRefreshMembers = useCallback(() => {
+ refetchMemberList();
+ refreshSearchMembers();
+ }, [refetchMemberList, refreshSearchMembers]);
+
const { isOpen: isOpenInvite, onOpen: onOpenInvite, onClose: onCloseInvite } = useDisclosure();
+ const { runAsync: onSyncMember, loading: isSyncing } = useRequest2(postSyncMembers, {
+ onSuccess: onRefreshMembers,
+ successToast: t('account_team:sync_member_success'),
+ errorToast: t('account_team:sync_member_failed')
+ });
+
+ const { ConfirmModal: ConfirmLeaveTeamModal, openConfirm: openLeaveConfirm } = useConfirm({
+ content: t('account_team:confirm_leave_team')
+ });
+ const { runAsync: onLeaveTeam } = useRequest2(delLeaveTeam, {
+ onSuccess() {
+ const defaultTeam = myTeams[0];
+ onSwitchTeam(defaultTeam.teamId);
+ },
+ errorToast: t('account_team:user_team_leave_team_failed')
+ });
+
const { ConfirmModal: ConfirmRemoveMemberModal, openConfirm: openRemoveMember } = useConfirm({
type: 'delete'
});
+ const { runAsync: onRemoveMember } = useRequest2(delRemoveMember, {
+ onSuccess: onRefreshMembers
+ });
const { ConfirmModal: ConfirmRestoreMemberModal, openConfirm: openRestoreMember } = useConfirm({
type: 'common',
@@ -76,59 +130,13 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
iconSrc: 'common/confirm/restoreTip',
iconColor: 'primary.500'
});
-
- const [searchText, setSearchText] = useState('');
- const isSyncMember = feConfigs.register_method?.includes('sync');
-
- const { data: searchMembersData } = useRequest2(
- async () => {
- if (!searchText) return Promise.resolve();
- return GetSearchUserGroupOrg(searchText, { members: true, orgs: false, groups: false });
- },
- {
- manual: false,
- throttleWait: 500,
- debounceWait: 200,
- refreshDeps: [searchText]
- }
- );
-
- const { runAsync: onLeaveTeam } = useRequest2(
- async () => {
- const defaultTeam = myTeams[0];
- // change to personal team
- onSwitchTeam(defaultTeam.teamId);
- return delLeaveTeam();
- },
- {
- onSuccess() {
- refetchTeams();
- refetchMembers();
- },
- errorToast: t('account_team:user_team_leave_team_failed')
- }
- );
- const { ConfirmModal: ConfirmLeaveTeamModal, openConfirm: openLeaveConfirm } = useConfirm({
- content: t('account_team:confirm_leave_team')
- });
-
- const { runAsync: onSyncMember, loading: isSyncing } = useRequest2(postSyncMembers, {
- onSuccess() {
- refetchMembers();
- },
- successToast: t('account_team:sync_member_success'),
- errorToast: t('account_team:sync_member_failed')
- });
-
- const { runAsync: onRestore, loading: isUpdateInvite } = useRequest2(postRestoreMember, {
- onSuccess() {
- refetchMembers();
- },
- successToast: t('common:user.team.invite.Accepted'),
+ const { runAsync: onRestore } = useRequest2(postRestoreMember, {
+ onSuccess: onRefreshMembers,
+ successToast: t('common:common.Success'),
errorToast: t('common:user.team.invite.Reject')
});
- const isLoading = isUpdateInvite || isSyncing;
+ const isLoading = loadingMembers || isSyncing;
const { EditModal: EditMemberNameModal, onOpenModal: openEditMemberName } = useEditTextarea({
title: t('account_team:edit_member'),
@@ -141,7 +149,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
defaultVal: memberName,
onSuccess: (newName: string) => {
return putUpdateMemberNameByManager(tmbId, newName).then(() => {
- refetchMembers();
+ onRefreshMembers();
});
},
onError: (err) => {
@@ -155,7 +163,6 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
return (
<>
- {isLoading && }
{Tabs}
@@ -237,7 +244,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
-
+
@@ -321,27 +328,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
}}
onClick={() => {
openRemoveMember(
- () => delRemoveMember(member.tmbId).then(refetchMembers),
- undefined,
- t('account_team:remove_tip', {
- username: member.memberName
- })
- )();
- }}
- />
- {
- openRemoveMember(
- () => delRemoveMember(member.tmbId).then(refetchMembers),
+ () => onRemoveMember(member.tmbId),
undefined,
t('account_team:remove_tip', {
username: member.memberName
@@ -385,7 +372,7 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
-
+
{isOpenInvite && userInfo?.team?.teamId && }
diff --git a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
index 1249bde30..15bd7dc78 100644
--- a/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
+++ b/projects/app/src/pageComponents/account/team/OrgManage/index.tsx
@@ -14,7 +14,7 @@ import {
Tr,
VStack
} from '@chakra-ui/react';
-import type { OrgListItemType, OrgType } from '@fastgpt/global/support/user/team/org/type';
+import type { OrgListItemType } from '@fastgpt/global/support/user/team/org/type';
import Avatar from '@fastgpt/web/components/common/Avatar';
import MyIcon from '@fastgpt/web/components/common/Icon';
import type { IconNameType } from '@fastgpt/web/components/common/Icon/type';
@@ -32,12 +32,9 @@ import { defaultOrgForm, type OrgFormType } from './OrgInfoModal';
import dynamic from 'next/dynamic';
import MyBox from '@fastgpt/web/components/common/MyBox';
import Path from '@/components/common/folder/Path';
-import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
-import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { delRemoveMember } from '@/web/support/user/team/api';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
-import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import useOrg from '@/web/support/user/team/org/hooks/useOrg';
const OrgInfoModal = dynamic(() => import('./OrgInfoModal'));
diff --git a/projects/app/src/pageComponents/account/team/context.tsx b/projects/app/src/pageComponents/account/team/context.tsx
index 0c29585b6..e5a30b16f 100644
--- a/projects/app/src/pageComponents/account/team/context.tsx
+++ b/projects/app/src/pageComponents/account/team/context.tsx
@@ -13,10 +13,8 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import type { TeamTmbItemType, TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import { getGroupList } from '@/web/support/user/team/group/api';
-import { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
-import { OrgType } from '@fastgpt/global/support/user/team/org/type';
+import { useRouter } from 'next/router';
const EditInfoModal = dynamic(() => import('./EditInfoModal'));
@@ -55,6 +53,8 @@ export const TeamContext = createContext({
export const TeamModalContextProvider = ({ children }: { children: ReactNode }) => {
const { t } = useTranslation();
+ const router = useRouter();
+
const [editTeamData, setEditTeamData] = useState();
const { userInfo, initUserInfo } = useUserStore();
@@ -96,10 +96,12 @@ export const TeamModalContextProvider = ({ children }: { children: ReactNode })
const { runAsync: onSwitchTeam, loading: isSwitchingTeam } = useRequest2(
async (teamId: string) => {
await putSwitchTeam(teamId);
- refetchMembers();
return initUserInfo();
},
{
+ onSuccess: () => {
+ router.reload();
+ },
errorToast: t('common:user.team.Switch Team Failed')
}
);
diff --git a/projects/app/src/web/support/user/team/group/api.ts b/projects/app/src/web/support/user/team/group/api.ts
index 7de009cfa..146510735 100644
--- a/projects/app/src/web/support/user/team/group/api.ts
+++ b/projects/app/src/web/support/user/team/group/api.ts
@@ -1,14 +1,19 @@
import { DELETE, GET, POST, PUT } from '@/web/common/api/request';
+import { GetGroupListBody } from '@fastgpt/global/support/permission/memberGroup/api';
import type {
GroupMemberItemType,
- MemberGroupListType
+ MemberGroupListItemType
} from '@fastgpt/global/support/permission/memberGroup/type';
import type {
postCreateGroupData,
putUpdateGroupData
} from '@fastgpt/global/support/user/team/group/api';
-export const getGroupList = () => GET('/proApi/support/user/team/group/list');
+export const getGroupList = (data: GetGroupListBody) =>
+ POST[]>('/proApi/support/user/team/group/list', data).then((res) => {
+ console.log(res);
+ return res;
+ });
export const postCreateGroup = (data: postCreateGroupData) =>
POST('/proApi/support/user/team/group/create', data);
From dd2f7bdcfd03440db0e7c305fbf77a2bad008764 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Wed, 26 Mar 2025 00:07:12 +0800
Subject: [PATCH 24/35] fix: ts (#4325)
* sync collection
* remove lock
* fix: ts
---
.../app/src/web/support/user/useUserStore.ts | 25 +------------------
1 file changed, 1 insertion(+), 24 deletions(-)
diff --git a/projects/app/src/web/support/user/useUserStore.ts b/projects/app/src/web/support/user/useUserStore.ts
index d0247f425..9f84ba982 100644
--- a/projects/app/src/web/support/user/useUserStore.ts
+++ b/projects/app/src/web/support/user/useUserStore.ts
@@ -3,7 +3,6 @@ import { create, devtools, persist, immer } from '@fastgpt/web/common/zustand';
import type { UserUpdateParams } from '@/types/user';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { getTokenLogin, putUserInfo } from '@/web/support/user/api';
-import type { MemberGroupListType } from '@fastgpt/global/support/permission/memberGroup/type';
import type { OrgType } from '@fastgpt/global/support/user/team/org/type';
import type { UserType } from '@fastgpt/global/support/user/type.d';
import type { FeTeamPlanStatusType } from '@fastgpt/global/support/wallet/sub/type';
@@ -26,10 +25,6 @@ type State = {
teamPlanStatus: FeTeamPlanStatusType | null;
initTeamPlanStatus: () => Promise;
- teamMemberGroups: MemberGroupListType;
- myGroups: MemberGroupListType;
- loadAndGetGroups: (init?: boolean) => Promise;
-
teamOrgs: OrgType[];
};
@@ -102,25 +97,7 @@ export const useUserStore = create()(
});
},
teamMemberGroups: [],
- teamOrgs: [],
- myGroups: [],
- loadAndGetGroups: async (init = false) => {
- if (!useSystemStore.getState()?.feConfigs?.isPlus) return [];
-
- const randomRefresh = Math.random() > 0.7;
- if (!randomRefresh && !init && get().teamMemberGroups.length)
- return Promise.resolve(get().teamMemberGroups);
-
- const res = await getGroupList();
- set((state) => {
- state.teamMemberGroups = res;
- state.myGroups = res.filter((item) =>
- item.members.map((i) => String(i.tmbId)).includes(String(state.userInfo?.team?.tmbId))
- );
- });
-
- return res;
- }
+ teamOrgs: []
})),
{
name: 'userStore',
From a17623d4ea6dc71193ba69688f5294eba787ae3d Mon Sep 17 00:00:00 2001
From: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Date: Wed, 26 Mar 2025 11:48:58 +0800
Subject: [PATCH 25/35] fix: group (#4330)
---
.../account/team/GroupManage/GroupInfoModal.tsx | 1 -
.../account/team/GroupManage/GroupManageMember.tsx | 7 +++++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
index d97b87590..22382e2c6 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupInfoModal.tsx
@@ -2,7 +2,6 @@ import { Input, HStack, ModalBody, Button, ModalFooter } from '@chakra-ui/react'
import MyModal from '@fastgpt/web/components/common/MyModal';
import Avatar from '@fastgpt/web/components/common/Avatar';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
-
import { useTranslation } from 'next-i18next';
import React from 'react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index 24341ba32..2b89afd6b 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -42,12 +42,10 @@ export type GroupFormType = {
// 3. Owner can add/remove admins
function GroupEditModal({
onClose,
- editGroupId,
group,
onSuccess
}: {
onClose: () => void;
- editGroupId?: string;
group: MemberGroupListItemType;
onSuccess: () => void;
}) {
@@ -61,10 +59,15 @@ function GroupEditModal({
const selectedMembersRef = useRef(null);
const [members, setMembers] = useState([]);
+ const editGroupId = useMemo(() => {
+ return group?._id;
+ }, [group]);
useMount(async () => {
+ console.log('aaaaaa');
if (editGroupId) {
const data = await getGroupMembers(editGroupId);
+ console.log(data);
setMembers(data);
}
});
From 484b87478c7af7cb1a97c18f9948c33b30b8c1f1 Mon Sep 17 00:00:00 2001
From: Archer <545436317@qq.com>
Date: Wed, 26 Mar 2025 21:38:52 +0800
Subject: [PATCH 26/35] perf: intro wrap (#4346)
* sync collection
* remove lock
* perf: intro wrap
---
projects/app/src/pageComponents/app/list/List.tsx | 4 +++-
projects/app/src/pageComponents/dataset/list/List.tsx | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/projects/app/src/pageComponents/app/list/List.tsx b/projects/app/src/pageComponents/app/list/List.tsx
index 234ba48b2..eed0693c6 100644
--- a/projects/app/src/pageComponents/app/list/List.tsx
+++ b/projects/app/src/pageComponents/app/list/List.tsx
@@ -212,7 +212,9 @@ const ListItem = () => {
fontSize={'xs'}
color={'myGray.500'}
>
- {app.intro || t('common:common.no_intro')}
+
+ {app.intro || t('common:common.no_intro')}
+
From 2ebb2ccc9c9e3ff0f9c39b9634146e172e5b9066 Mon Sep 17 00:00:00 2001
From: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Date: Wed, 26 Mar 2025 22:10:03 +0800
Subject: [PATCH 27/35] pref: member list (#4344)
* chore: search member new api
* chore: permission
* fix: ts error
* fix: member modal
---
.../global/support/user/team/org/api.d.ts | 3 +-
packages/global/support/user/team/type.d.ts | 26 +-
.../permission/MemberManager/MemberModal.tsx | 159 +++++------
.../team/GroupManage/GroupManageMember.tsx | 170 ++++++------
.../GroupManage/GroupTransferOwnerModal.tsx | 80 +++---
.../account/team/MemberTable.tsx | 257 ++++++++++--------
.../account/team/OrgManage/OrgInfoModal.tsx | 7 +-
.../team/OrgManage/OrgMemberManageModal.tsx | 78 ++----
.../account/team/OrgManage/OrgMoveModal.tsx | 4 +-
.../account/team/OrgManage/OrgTree.tsx | 15 +-
.../account/team/OrgManage/index.tsx | 159 ++++-------
.../pageComponents/account/team/context.tsx | 43 +--
projects/app/src/web/support/user/team/api.ts | 12 +-
.../app/src/web/support/user/team/org/api.ts | 7 +-
.../support/user/team/org/hooks/useOrg.tsx | 36 ++-
15 files changed, 508 insertions(+), 548 deletions(-)
diff --git a/packages/global/support/user/team/org/api.d.ts b/packages/global/support/user/team/org/api.d.ts
index feaded6f1..dee9bf4d8 100644
--- a/packages/global/support/user/team/org/api.d.ts
+++ b/packages/global/support/user/team/org/api.d.ts
@@ -1,9 +1,8 @@
-// orgId, pathid, path === null ===> root org
export type postCreateOrgData = {
name: string;
description?: string;
avatar?: string;
- path?: string;
+ orgId?: string;
};
export type putUpdateOrgMembersData = {
diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts
index c715d93b5..597eaf662 100644
--- a/packages/global/support/user/team/type.d.ts
+++ b/packages/global/support/user/team/type.d.ts
@@ -70,7 +70,13 @@ export type TeamTmbItemType = {
permission: TeamPermission;
} & ThirdPartyAccountType;
-export type TeamMemberItemType = {
+export type TeamMemberItemType<
+ Options extends {
+ withPermission?: boolean;
+ withOrgs?: boolean;
+ withGroupRole?: boolean;
+ } = { withPermission: true; withOrgs: true; withGroupRole: false }
+> = {
userId: string;
tmbId: string;
teamId: string;
@@ -78,12 +84,24 @@ export type TeamMemberItemType = {
avatar: string;
role: `${TeamMemberRoleEnum}`;
status: `${TeamMemberStatusEnum}`;
- permission: TeamPermission;
contact?: string;
createTime: Date;
updateTime?: Date;
- orgs?: string[]; // full path name, pattern: /teamName/orgname1/orgname2
-};
+} & (Options extends { withPermission: true }
+ ? {
+ permission: TeamPermission;
+ }
+ : {}) &
+ (Options extends { withOrgs: true }
+ ? {
+ orgs?: string[]; // full path name, pattern: /teamName/orgname1/orgname2
+ }
+ : {}) &
+ (Options extends { withGroupRole: true }
+ ? {
+ groupRole?: `${GroupMemberRole}`;
+ }
+ : {});
export type TeamTagItemType = {
label: string;
diff --git a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
index 02b762893..11f70e921 100644
--- a/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
+++ b/projects/app/src/components/support/permission/MemberManager/MemberModal.tsx
@@ -19,7 +19,7 @@ import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import { useMemo, useRef, useState } from 'react';
+import { useEffect, useMemo, useRef, useState } from 'react';
import PermissionSelect from './PermissionSelect';
import PermissionTags from './PermissionTags';
import {
@@ -39,6 +39,7 @@ import { GetSearchUserGroupOrg } from '@/web/support/user/api';
import useOrg from '@/web/support/user/team/org/hooks/useOrg';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
+import _ from 'lodash';
const HoverBoxStyle = {
bgColor: 'myGray.50',
@@ -55,7 +56,6 @@ function MemberModal({
const { t } = useTranslation();
const { userInfo } = useUserStore();
const collaboratorList = useContextSelector(CollaboratorContext, (v) => v.collaboratorList);
- const [searchText, setSearchText] = useState('');
const [filterClass, setFilterClass] = useState<'member' | 'org' | 'group'>();
const {
paths,
@@ -63,18 +63,35 @@ function MemberModal({
members: orgMembers,
MemberScrollData: OrgMemberScrollData,
onPathClick,
- orgs
- } = useOrg({ getPermission: false });
+ orgs,
+ searchKey,
+ setSearchKey
+ } = useOrg({ withPermission: false });
- const { data: members, ScrollData: TeamMemberScrollData } = useScrollPagination(getTeamMembers, {
- pageSize: 15
+ const {
+ data: members,
+ ScrollData: TeamMemberScrollData,
+ refreshList
+ } = useScrollPagination(getTeamMembers, {
+ pageSize: 15,
+ params: {
+ withPermission: true,
+ withOrgs: true,
+ status: 'active',
+ searchKey
+ }
});
- const { data: groups = [], loading: loadingGroupsAndOrgs } = useRequest2(
+ const {
+ data: groups = [],
+ loading: loadingGroupsAndOrgs,
+ runAsync: refreshGroups
+ } = useRequest2(
async () => {
if (!userInfo?.team?.teamId) return [];
return getGroupList({
- withMembers: false
+ withMembers: false,
+ searchKey
});
},
{
@@ -83,53 +100,20 @@ function MemberModal({
}
);
- const { data: searchedData } = useRequest2(() => GetSearchUserGroupOrg(searchText), {
- manual: false,
- throttleWait: 500,
- debounceWait: 200,
- refreshDeps: [searchText]
- });
+ const search = _.debounce(() => {
+ refreshList();
+ refreshGroups();
+ }, 200);
+
+ useEffect(search, [searchKey]);
const [selectedOrgList, setSelectedOrgIdList] = useState([]);
- const filterOrgs: (OrgListItemType & { count?: number })[] = useMemo(() => {
- if (searchText && searchedData) {
- const orgids = searchedData.orgs.map((item) => item._id);
- return orgs.filter((org) => orgids.includes(String(org._id)));
- }
- return orgs
- .filter((org) => org.path !== '')
- .map((org) => ({
- ...org,
- count: org.total
- }));
- }, [searchText, orgs, searchedData]);
-
const [selectedMemberList, setSelectedMemberList] = useState<
Omit[]
>([]);
- const filterMembers = useMemo(() => {
- if (searchText) {
- return searchedData?.members || [];
- }
-
- return members;
- }, [searchText, members, searchedData?.members]);
const [selectedGroupList, setSelectedGroupList] = useState[]>([]);
- const filterGroups = useMemo(() => {
- if (searchText) {
- return searchedData?.groups.map((item) => ({
- groupName: item.name,
- _id: item.id,
- ...item
- }));
- }
- if (!searchText && filterClass !== 'group') return [];
-
- return groups;
- }, [searchText, filterClass, groups, searchedData]);
-
const permissionList = useContextSelector(CollaboratorContext, (v) => v.permissionList);
const getPerLabelList = useContextSelector(CollaboratorContext, (v) => v.getPerLabelList);
const [selectedPermission, setSelectedPermission] = useState(
@@ -225,12 +209,12 @@ function MemberModal({
setSearchText(e.target.value)}
+ onChange={(e) => setSearchKey(e.target.value)}
/>
{/* Entry */}
- {!searchText && !filterClass && (
+ {!searchKey && !filterClass && (
<>
{entryList.current.map((item) => {
return (
@@ -257,7 +241,7 @@ function MemberModal({
)}
{/* Path */}
- {!searchText && filterClass && (
+ {!searchKey && filterClass && (
)}
- {(filterClass === 'member' || (searchText && filterMembers.length > 0)) &&
+ {(filterClass === 'member' || searchKey) &&
(() => {
- const members = filterMembers?.map((member) => {
+ const Members = members?.map((member) => {
const onChange = () => {
setSelectedMemberList((state) => {
if (state.find((v) => v.tmbId === member.tmbId)) {
@@ -315,8 +299,8 @@ function MemberModal({
/>
);
});
- return searchText ? (
- members
+ return searchKey ? (
+ Members
) : (
- {members}
+ {Members}
);
})()}
- {(filterClass === 'org' || searchText) &&
+ {(filterClass === 'org' || searchKey) &&
(() => {
- const orgs = filterOrgs?.map((org) => {
+ const Orgs = orgs?.map((org) => {
const onChange = () => {
setSelectedOrgIdList((state) => {
if (state.find((v) => v._id === org._id)) {
@@ -356,18 +340,18 @@ function MemberModal({
pointerEvents="none"
/>
-
+
{org.name}
- {org.count && (
+ {org.total && (
<>
- {org.count}
+ {org.total}
>
)}
- {org.count && (
+ {org.total && (
);
});
- return searchText ? (
- orgs
+ return searchKey ? (
+ Orgs
) : (
- {orgs}
+ {Orgs}
{orgMembers.map((member) => {
return (
);
})()}
- {filterGroups?.map((group) => {
- const onChange = () => {
- setSelectedGroupList((state) => {
- if (state.find((v) => v._id === group._id)) {
- return state.filter((v) => v._id !== group._id);
- }
- return [...state, group];
- });
- };
- const collaborator = collaboratorList?.find((v) => v.groupId === group._id);
- return (
- v._id === group._id)}
- />
- );
- })}
+ {(filterClass === 'group' || searchKey) &&
+ groups?.map((group) => {
+ const onChange = () => {
+ setSelectedGroupList((state) => {
+ if (state.find((v) => v._id === group._id)) {
+ return state.filter((v) => v._id !== group._id);
+ }
+ return [...state, group];
+ });
+ };
+ const collaborator = collaboratorList?.find((v) => v.groupId === group._id);
+ return (
+ v._id === group._id)}
+ />
+ );
+ })}
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
index 2b89afd6b..2ce0f2c23 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupManageMember.tsx
@@ -1,34 +1,25 @@
-import {
- Box,
- ModalBody,
- Flex,
- Button,
- ModalFooter,
- Checkbox,
- Grid,
- HStack
-} from '@chakra-ui/react';
+import { Box, ModalBody, Flex, Button, ModalFooter, Grid, HStack } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import Avatar from '@fastgpt/web/components/common/Avatar';
import Tag from '@fastgpt/web/components/common/Tag';
import { useTranslation } from 'next-i18next';
-import React, { useMemo, useRef, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { useContextSelector } from 'use-context-selector';
-import { TeamContext } from '../context';
-import { getGroupMembers, putUpdateGroup } from '@/web/support/user/team/group/api';
+import { putUpdateGroup } from '@/web/support/user/team/group/api';
import { GroupMemberRole } from '@fastgpt/global/support/permission/memberGroup/constant';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
-import {
- GroupMemberItemType,
- MemberGroupListItemType
-} from '@fastgpt/global/support/permission/memberGroup/type';
-import { useMount } from 'ahooks';
+import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
+import { getTeamMembers } from '@/web/support/user/team/api';
+import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
+import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import _ from 'lodash';
+import MemberItemCard from '@/components/support/permission/MemberManager/MemberItemCard';
export type GroupFormType = {
members: {
@@ -53,42 +44,65 @@ function GroupEditModal({
const { userInfo } = useUserStore();
const { toast } = useToast();
- const allMembers = useContextSelector(TeamContext, (v) => v.members);
- const MemberScrollData = useContextSelector(TeamContext, (v) => v.MemberScrollData);
- const [hoveredMemberId, setHoveredMemberId] = useState();
+ const [searchKey, setSearchKey] = useState('');
+ const [selected, setSelected] = useState<
+ { name: string; tmbId: string; avatar: string; role: `${GroupMemberRole}` }[]
+ >([]);
- const selectedMembersRef = useRef(null);
- const [members, setMembers] = useState([]);
- const editGroupId = useMemo(() => {
- return group?._id;
- }, [group]);
+ const {
+ data: allMembers = [],
+ ScrollData: MemberScrollData,
+ refreshList
+ } = useScrollPagination<
+ any,
+ PaginationResponse>
+ >(getTeamMembers, {
+ pageSize: 20,
+ params: {
+ status: 'active',
+ withOrgs: true,
+ searchKey
+ }
+ });
+ const refetchMemberList = _.debounce(refreshList, 200);
- useMount(async () => {
- console.log('aaaaaa');
- if (editGroupId) {
- const data = await getGroupMembers(editGroupId);
- console.log(data);
- setMembers(data);
+ useEffect(() => refetchMemberList, [searchKey]);
+
+ const groupId = useMemo(() => String(group._id), [group._id]);
+
+ const { data: groupMembers = [], ScrollData: GroupScrollData } = useScrollPagination<
+ any,
+ PaginationResponse<
+ TeamMemberItemType<{ withOrgs: true; withPermission: true; withGroupRole: true }>
+ >
+ >(getTeamMembers, {
+ pageSize: 100000,
+ params: {
+ groupId: groupId
}
});
- const [searchKey, setSearchKey] = useState('');
- const filtered = useMemo(() => {
- return [
- ...allMembers.filter((member) => {
- if (member.memberName.toLowerCase().includes(searchKey.toLowerCase())) return true;
- return false;
- })
- ];
- }, [searchKey, allMembers]);
+ useEffect(() => {
+ if (!groupId) return;
+ setSelected(
+ groupMembers.map((item) => ({
+ name: item.memberName,
+ tmbId: item.tmbId,
+ avatar: item.avatar,
+ role: (item.groupRole ?? 'member') as `${GroupMemberRole}`
+ }))
+ );
+ }, [groupId, groupMembers]);
+
+ const [hoveredMemberId, setHoveredMemberId] = useState();
const { runAsync: onUpdate, loading: isLoadingUpdate } = useRequest2(
async () => {
- if (!editGroupId || !members.length) return;
+ if (!group._id || !groupMembers.length) return;
return putUpdateGroup({
- groupId: editGroupId,
- memberList: members
+ groupId: group._id,
+ memberList: selected
});
},
{
@@ -97,18 +111,21 @@ function GroupEditModal({
);
const isSelected = (memberId: string) => {
- return members.find((item) => item.tmbId === memberId);
+ return selected.find((item) => item.tmbId === memberId);
};
const myRole = useMemo(() => {
if (userInfo?.team.permission.hasManagePer) {
return 'owner';
}
- return members.find((item) => item.tmbId === userInfo?.team.tmbId)?.role ?? 'member';
- }, [members, userInfo]);
+ return groupMembers.find((item) => item.tmbId === userInfo?.team.tmbId)?.groupRole ?? 'member';
+ }, [groupMembers, userInfo]);
const handleToggleSelect = (memberId: string) => {
- if (myRole === 'owner' && memberId === members.find((item) => item.role === 'owner')?.tmbId) {
+ if (
+ myRole === 'owner' &&
+ memberId === groupMembers.find((item) => item.role === 'owner')?.tmbId
+ ) {
toast({
title: t('user:team.group.toast.can_not_delete_owner'),
status: 'error'
@@ -118,18 +135,18 @@ function GroupEditModal({
if (
myRole === 'admin' &&
- members.find((item) => String(item.tmbId) === memberId)?.role !== 'member'
+ selected.find((item) => String(item.tmbId) === memberId)?.role !== 'member'
) {
return;
}
if (isSelected(memberId)) {
- setMembers(members.filter((item) => item.tmbId !== memberId));
+ setSelected(selected.filter((item) => item.tmbId !== memberId));
} else {
const member = allMembers.find((m) => m.tmbId === memberId);
if (!member) return;
- setMembers([
- ...members,
+ setSelected([
+ ...selected,
{
name: member.memberName,
avatar: member.avatar,
@@ -142,14 +159,14 @@ function GroupEditModal({
const handleToggleAdmin = (memberId: string) => {
if (myRole === 'owner' && isSelected(memberId)) {
- const oldRole = members.find((item) => item.tmbId === memberId)?.role;
+ const oldRole = groupMembers.find((item) => item.tmbId === memberId)?.groupRole;
if (oldRole === 'admin') {
- setMembers(
- members.map((item) => (item.tmbId === memberId ? { ...item, role: 'member' } : item))
+ setSelected(
+ selected.map((item) => (item.tmbId === memberId ? { ...item, role: 'member' } : item))
);
} else {
- setMembers(
- members.map((item) => (item.tmbId === memberId ? { ...item, role: 'admin' } : item))
+ setSelected(
+ selected.map((item) => (item.tmbId === memberId ? { ...item, role: 'admin' } : item))
);
}
}
@@ -184,37 +201,24 @@ function GroupEditModal({
}}
/>
- {filtered.map((member) => {
+ {allMembers.map((member) => {
return (
- handleToggleSelect(member.tmbId)}
- >
- }
- />
-
- {member.memberName}
-
+ name={member.memberName}
+ onChange={() => handleToggleSelect(member.tmbId)}
+ isChecked={!!isSelected(member.tmbId)}
+ orgs={member.orgs}
+ />
);
})}
- {t('common:chosen') + ': ' + members.length}
-
- {members.map((member) => {
+ {t('common:chosen') + ': ' + selected.length}
+
+ {selected.map((member) => {
return (
setHoveredMemberId(member.tmbId)}
@@ -284,7 +288,7 @@ function GroupEditModal({
);
})}
-
+
diff --git a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
index 48741fc91..9915b6afc 100644
--- a/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
+++ b/projects/app/src/pageComponents/account/team/GroupManage/GroupTransferOwnerModal.tsx
@@ -15,12 +15,16 @@ import Avatar from '@fastgpt/web/components/common/Avatar';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
import { TeamContext } from '../context';
import { useContextSelector } from 'use-context-selector';
import { MemberGroupListItemType } from '@fastgpt/global/support/permission/memberGroup/type';
import { GetSearchUserGroupOrg } from '@/web/support/user/api';
import { Omit } from '@fastgpt/web/components/common/DndDrag';
+import { getTeamMembers } from '@/web/support/user/team/api';
+import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import _ from 'lodash';
export function ChangeOwnerModal({
group,
@@ -33,22 +37,24 @@ export function ChangeOwnerModal({
}) {
const { t } = useTranslation();
- const [inputValue, setInputValue] = React.useState('');
- const { data: searchedData } = useRequest2(
- async () => {
- if (!inputValue) return;
- return GetSearchUserGroupOrg(inputValue);
- },
+ const [searchKey, setSearchKey] = React.useState('');
+
+ const {
+ data: members = [],
+ ScrollData: MemberScrollData,
+ refreshList
+ } = useScrollPagination>>(
+ getTeamMembers,
{
- manual: false,
- refreshDeps: [inputValue],
- throttleWait: 500,
- debounceWait: 200
+ pageSize: 20,
+ params: {
+ searchKey: searchKey
+ }
}
);
- const { members: allMembers } = useContextSelector(TeamContext, (v) => v);
- const memberList = searchedData ? searchedData.members : allMembers;
+ const search = _.debounce(refreshList, 500);
+ useEffect(() => search, [searchKey]);
const {
isOpen: isOpenMemberListMenu,
@@ -108,9 +114,9 @@ export function ChangeOwnerModal({
)}
{
- setInputValue(e.target.value);
+ setSearchKey(e.target.value);
setSelectedMember(null);
}}
onFocus={() => {
@@ -120,7 +126,7 @@ export function ChangeOwnerModal({
{...(selectedMember && { pl: '10' })}
/>
- {isOpenMemberListMenu && memberList.length > 0 && (
+ {isOpenMemberListMenu && members.length > 0 && (
- {memberList.map((item) => (
- {
- setInputValue(item.memberName);
- setSelectedMember(item);
- onCloseMemberListMenu();
- }}
- >
-
-
- {item.memberName}
-
-
- ))}
+
+ {members.map((item) => (
+ {
+ setSearchKey(item.memberName);
+ setSelectedMember(item);
+ onCloseMemberListMenu();
+ }}
+ >
+
+
+ {item.memberName}
+
+
+ ))}
+
)}
diff --git a/projects/app/src/pageComponents/account/team/MemberTable.tsx b/projects/app/src/pageComponents/account/team/MemberTable.tsx
index cb8c5e4ea..b7795681d 100644
--- a/projects/app/src/pageComponents/account/team/MemberTable.tsx
+++ b/projects/app/src/pageComponents/account/team/MemberTable.tsx
@@ -41,12 +41,15 @@ import {
import format from 'date-fns/format';
import OrgTags from '@/components/support/user/team/OrgTags';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
-import { useCallback, useState } from 'react';
+import { useCallback, useEffect, useState } from 'react';
import { downloadFetch } from '@/web/common/system/utils';
import { TeamMemberItemType } from '@fastgpt/global/support/user/team/type';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import _ from 'lodash';
+import MySelect from '@fastgpt/web/components/common/MySelect';
const InviteModal = dynamic(() => import('./Invite/InviteModal'));
const TeamTagModal = dynamic(() => import('@/components/support/user/team/TeamTagModal'));
@@ -55,11 +58,26 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
const { t } = useTranslation();
const { toast } = useToast();
+ const statusOptions = [
+ {
+ label: t('common:common.All'),
+ value: undefined
+ },
+ {
+ label: t('common:user.team.member.active'),
+ value: 'active'
+ },
+ {
+ label: t('account_team:leave'),
+ value: 'inactive'
+ }
+ ];
const { userInfo } = useUserStore();
const { feConfigs } = useSystemStore();
const isSyncMember = feConfigs?.register_method?.includes('sync');
const { myTeams, onSwitchTeam } = useContextSelector(TeamContext, (v) => v);
+ const [status, setStatus] = useState();
const {
isOpen: isOpenTeamTagsAsync,
@@ -68,35 +86,36 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
} = useDisclosure();
// member action
+ const [searchKey, setSearchKey] = useState('');
const {
data: members = [],
isLoading: loadingMembers,
refreshList: refetchMemberList,
ScrollData: MemberScrollData
- } = useScrollPagination(getTeamMembers, {
+ } = useScrollPagination<
+ any,
+ PaginationResponse>
+ >(getTeamMembers, {
pageSize: 20,
params: {
- withLeaved: true
+ status,
+ withPermission: true,
+ withOrgs: true,
+ searchKey
}
});
- const [searchText, setSearchText] = useState('');
- const { data: searchMembersData, run: refreshSearchMembers } = useRequest2(
- async () => {
- if (!searchText) return Promise.resolve();
- return GetSearchUserGroupOrg(searchText, { members: true, orgs: false, groups: false });
- },
- {
- manual: false,
- throttleWait: 500,
- refreshDeps: [searchText]
- }
- );
+ const refreshList = _.debounce(() => {
+ refetchMemberList();
+ }, 200);
+
+ useEffect(() => {
+ refreshList();
+ }, [searchKey, status]);
const onRefreshMembers = useCallback(() => {
refetchMemberList();
- refreshSearchMembers();
- }, [refetchMemberList, refreshSearchMembers]);
+ }, [refetchMemberList]);
const { isOpen: isOpenInvite, onOpen: onOpenInvite, onClose: onCloseInvite } = useDisclosure();
@@ -166,10 +185,13 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
{Tabs}
+
+ setStatus(v)} />
+
setSearchText(e.target.value)}
+ onChange={(e) => setSearchKey(e.target.value)}
/>
{userInfo?.team.permission.hasManagePer && feConfigs?.show_team_chat && (
@@ -264,107 +286,104 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
- {(searchText && searchMembersData ? searchMembersData.members : members).map(
- (member) => (
-
- |
-
- | | | | | |