v4.6-3 (#471)

.github/workflows/fastgpt-image-personal.yml

@@ -0,0 +1,52 @@
+name: Build FastGPT images in Personal warehouse
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'projects/app/**'
+      - 'packages/**'
+    branches:
+      - 'main'
+jobs:
+  build-fastgpt-images:
+    runs-on: ubuntu-20.04
+    if: github.repository != 'labring/FastGPT'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GH_PAT }}
+      - name: Set DOCKER_REPO_TAGGED based on branch or tag
+        run: |
+          echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt:latest" >> $GITHUB_ENV
+      - name: Build and publish image for main branch or tag push event
+        env:
+          DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
+        run: |
+          docker buildx build \
+          --build-arg name=app \
+          --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
+          --label "org.opencontainers.image.description=fastgpt image" \
+          --push \
+          --cache-from=type=local,src=/tmp/.buildx-cache \
+          --cache-to=type=local,dest=/tmp/.buildx-cache \
+          -t ${DOCKER_REPO_TAGGED} \
+          -f Dockerfile \
+          .

.github/workflows/fastgpt-image.yml

@@ -5,8 +5,6 @@ on:
     paths:
       - 'projects/app/**'
       - 'packages/**'
-    branches:
-      - 'main'
     tags:
       - 'v*.*.*'
 jobs:
@@ -53,9 +51,8 @@ jobs:
           docker buildx build \
           --build-arg name=app \
           --platform linux/amd64,linux/arm64 \
-          --label "org.opencontainers.image.source=  https://github.com/  ${{ github.repository_owner }}/FastGPT" \
+          --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
           --label "org.opencontainers.image.description=fastgpt image" \
-          --label "org.opencontainers.image.licenses=Apache" \
           --push \
           --cache-from=type=local,src=/tmp/.buildx-cache \
           --cache-to=type=local,dest=/tmp/.buildx-cache \

.github/workflows/preview-image.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           driver-opts: network=host
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -48,6 +48,7 @@ jobs:
           --label "org.opencontainers.image.source=  https://github.com/  ${{ github.repository_owner }}/FastGPT" \
           --label "org.opencontainers.image.description=fastgpt-pr image" \
           --label "org.opencontainers.image.licenses=Apache" \
+          --push \
           --cache-from=type=local,src=/tmp/.buildx-cache \
           --cache-to=type=local,dest=/tmp/.buildx-cache \
           -t ${DOCKER_REPO_TAGGED} \

docSite/content/docs/installation/upgrading/46.md

@@ -7,20 +7,20 @@ toc: true
 weight: 836
 ---
 
-未正式发布。
+# V4.6 版本加入了简单的团队功能，可以邀请其他用户进来管理资源。该版本升级后无法执行旧的升级脚本，且无法回退。
 
-V4.6 版本加入了简单的团队功能，可以邀请其他用户进来管理资源。该版本升级后无法执行旧的升级脚本，且无法回退。
+# 1. 更新镜像并变更配置文件
 
-## 1. 更新镜像并变更配置文件
+更新镜像至 latest 或者 v4.6 版本。商业版镜像更新至 V0.2.1
-
-更新镜像至 latest 或者 v4.6 版本。商业版镜像更新至 V0.2.
 
 最新配置可参考: [V46版本最新 config.json](/docs/development/configuration)，商业镜像配置文件也更新，参考最新的飞书文档。
 
 
-## 2. 执行初始化 API
+# 2. 执行初始化 API
 
-发起 1 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名）
+发起 2 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名）
+
+**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可，需要注意的是，需确保initv46成功后，在执行initv46-2**
 
 1. https://xxxxx/api/admin/initv46
 
@@ -30,16 +30,25 @@ curl --location --request POST 'https://{{host}}/api/admin/initv46' \
 --header 'Content-Type: application/json'
 ```
 
+2. https://xxxxx/api/admin/initv46-2
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv46-2' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
 初始化内容：
 1. 创建默认团队
 2. 初始化 Mongo 所有资源的团队字段
 3. 初始化 Pg 的字段
-
+4. 初始化 Mongo Data
-**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可**
 
 
-## 功能介绍
+# V4.6功能介绍
-
-### Fast GPT V4.6
 
 1. 新增 - 团队空间
+2. 新增 - 多路向量（多个向量映射一组数据）
+3. 新增 - tts语音
+4. 线上环境新增 - ReRank向量召回，提高召回精度
+5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈

packages/global/common/string/textSplitter.ts

@@ -0,0 +1,131 @@
+import { getErrText } from '../error/utils';
+import { countPromptTokens } from './tiktoken';
+
+/**
+ * text split into chunks
+ * maxLen - one chunk len. max: 3500
+ * overlapLen - The size of the before and after Text
+ * maxLen > overlapLen
+ * markdown
+ */
+export const splitText2Chunks = (props: { text: string; maxLen: number; overlapLen?: number }) => {
+  const { text = '', maxLen, overlapLen = Math.floor(maxLen * 0.2) } = props;
+  const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
+
+  const stepReg: Record<number, RegExp> = {
+    0: /^(#\s[^\n]+)\n/gm,
+    1: /^(##\s[^\n]+)\n/gm,
+    2: /^(###\s[^\n]+)\n/gm,
+    3: /^(####\s[^\n]+)\n/gm,
+
+    4: /(\n\n)/g,
+    5: /([\n])/g,
+    6: /[。]|(?!<[^a-zA-Z])\.\s/g,
+    7: /([！？]|!\s|\?\s)/g,
+    8: /([；]|;\s)/g,
+    9: /([，]|,\s)/g
+  };
+
+  const splitTextRecursively = ({
+    text = '',
+    step,
+    lastChunk,
+    overlayChunk
+  }: {
+    text: string;
+    step: number;
+    lastChunk: string;
+    overlayChunk: string;
+  }) => {
+    if (text.length <= maxLen) {
+      return [text];
+    }
+    const reg = stepReg[step];
+    const isMarkdownSplit = step < 4;
+
+    if (!reg) {
+      // use slice-maxLen to split text
+      const chunks: string[] = [];
+      let chunk = '';
+      for (let i = 0; i < text.length; i += maxLen - overlapLen) {
+        chunk = text.slice(i, i + maxLen);
+        chunks.push(chunk);
+      }
+      return chunks;
+    }
+
+    // split text by special char
+    const splitTexts = text
+      .replace(reg, isMarkdownSplit ? `${tempMarker}$1` : `$1${tempMarker}`)
+      .split(`${tempMarker}`)
+      .filter((part) => part);
+
+    let chunks: string[] = [];
+    for (let i = 0; i < splitTexts.length; i++) {
+      let text = splitTexts[i];
+      let chunkToken = countPromptTokens(lastChunk, '');
+      const textToken = countPromptTokens(text, '');
+
+      // next chunk is too large / new chunk is too large(The current chunk must be smaller than maxLen)
+      if (textToken >= maxLen || chunkToken + textToken > maxLen * 1.4) {
+        // last chunk is too large, push it to chunks, not add to next chunk
+        if (chunkToken > maxLen * 0.7) {
+          chunks.push(lastChunk);
+          lastChunk = '';
+          overlayChunk = '';
+        }
+        // chunk is small, insert to next chunks
+        const innerChunks = splitTextRecursively({
+          text,
+          step: step + 1,
+          lastChunk,
+          overlayChunk
+        });
+        if (innerChunks.length === 0) continue;
+        chunks = chunks.concat(innerChunks);
+        lastChunk = '';
+        overlayChunk = '';
+        continue;
+      }
+
+      // size less than maxLen, push text to last chunk
+      lastChunk += text;
+      chunkToken += textToken; // Definitely less than 1.4 * maxLen
+
+      // size over lapLen, push it to next chunk
+      if (
+        overlapLen !== 0 &&
+        !isMarkdownSplit &&
+        chunkToken >= maxLen - overlapLen &&
+        textToken < overlapLen
+      ) {
+        overlayChunk += text;
+      }
+      if (chunkToken >= maxLen) {
+        chunks.push(lastChunk);
+        lastChunk = overlayChunk;
+        overlayChunk = '';
+      }
+    }
+
+    /* If the last chunk is independent, it needs to be push chunks. */
+    if (lastChunk && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastChunk)) {
+      chunks.push(lastChunk);
+    }
+
+    return chunks;
+  };
+
+  try {
+    const chunks = splitTextRecursively({ text, step: 0, lastChunk: '', overlayChunk: '' });
+
+    const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
+
+    return {
+      chunks,
+      tokens
+    };
+  } catch (err) {
+    throw new Error(getErrText(err));
+  }
+};

packages/global/common/string/tiktoken/index.ts

@@ -1,8 +1,8 @@
 /* Only the token of gpt-3.5-turbo is used */
-import type { ChatItemType } from '@fastgpt/global/core/chat/type';
+import type { ChatItemType } from '../../../core/chat/type';
 import { Tiktoken } from 'js-tiktoken/lite';
-import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
+import { adaptChat2GptMessages } from '../../../core/chat/adapt';
-import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constant';
+import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
 import encodingJson from './cl100k_base.json';
 
 /* init tikToken obj */
@@ -55,17 +55,6 @@ export function countMessagesTokens({ messages }: { messages: ChatItemType[] })
   return totalTokens;
 }
 
-export function sliceTextByTokens({ text, length }: { text: string; length: number }) {
-  const enc = getTikTokenEnc();
-
-  try {
-    const encodeText = enc.encode(text);
-    return enc.decode(encodeText.slice(0, length));
-  } catch (error) {
-    return text.slice(0, length);
-  }
-}
-
 /* slice messages from top to bottom by maxTokens */
 export function sliceMessagesTB({
   messages,

packages/global/common/string/tiktoken/type.d.ts

@@ -0,0 +1,5 @@
+import type { Tiktoken } from 'js-tiktoken';
+
+declare global {
+  var TikToken: Tiktoken;
+}

packages/global/common/string/tools.ts

@@ -1,13 +1,15 @@
 import crypto from 'crypto';
 
+/* check string is a web link */
 export function strIsLink(str?: string) {
   if (!str) return false;
   if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
   return false;
 }
 
-export const hashStr = (psw: string) => {
+/* hash string */
-  return crypto.createHash('sha256').update(psw).digest('hex');
+export const hashStr = (str: string) => {
+  return crypto.createHash('sha256').update(str).digest('hex');
 };
 
 /* simple text, remove chinese space and extra \n */
@@ -20,3 +22,16 @@ export const simpleText = (text: string) => {
 
   return text;
 };
+
+/* 
+    replace {{variable}} to value
+*/
+export function replaceVariable(text: string, obj: Record<string, string | number>) {
+  for (const key in obj) {
+    const val = obj[key];
+    if (!['string', 'number'].includes(typeof val)) continue;
+
+    text = text.replace(new RegExp(`{{(${key})}}`, 'g'), String(val));
+  }
+  return text || '';
+}

packages/global/core/ai/api.d.ts

@@ -0,0 +1,5 @@
+export type PostReRankProps = {
+  query: string;
+  inputs: { id: string; text: string }[];
+};
+export type PostReRankResponse = { id: string; score: number }[];

packages/global/core/chat/adapt.ts

@@ -1,7 +1,7 @@
-import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
+import type { ChatItemType } from '../../core/chat/type.d';
-import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import { ChatRoleEnum } from '../../core/chat/constants';
-import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constant';
+import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constant';
-import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
+import type { ChatMessageItemType } from '../../core/ai/type.d';
 
 const chat2Message = {
   [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,

packages/global/core/dataset/api.d.ts

@@ -0,0 +1,20 @@
+import { DatasetDataIndexItemType } from './type';
+
+/* ================= dataset ===================== */
+
+/* ================= collection ===================== */
+
+/* ================= data ===================== */
+export type PgSearchRawType = {
+  id: string;
+  team_id: string;
+  tmb_id: string;
+  collection_id: string;
+  data_id: string;
+  score: number;
+};
+export type PushDatasetDataChunkProps = {
+  q: string; // embedding content
+  a?: string; // bonus content
+  indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
+};

packages/global/core/dataset/constant.ts

@@ -36,29 +36,54 @@ export const DatasetCollectionTypeMap = {
   }
 };
 
-export enum TrainingModeEnum {
+export enum DatasetDataIndexTypeEnum {
-  'qa' = 'qa',
+  chunk = 'chunk',
-  'index' = 'index'
+  qa = 'qa',
+  summary = 'summary',
+  hypothetical = 'hypothetical',
+  custom = 'custom'
 }
-export const TrainingTypeMap = {
+export const DatasetDataIndexTypeMap = {
-  [TrainingModeEnum.qa]: 'qa',
+  [DatasetDataIndexTypeEnum.chunk]: {
-  [TrainingModeEnum.index]: 'index'
+    name: 'dataset.data.indexes.chunk'
-};
-
-export enum DatasetSpecialIdEnum {
-  manual = 'manual',
-  mark = 'mark'
-}
-export const datasetSpecialIdMap = {
-  [DatasetSpecialIdEnum.manual]: {
-    name: 'kb.Manual Data',
-    sourceName: 'kb.Manual Input'
   },
-  [DatasetSpecialIdEnum.mark]: {
+  [DatasetDataIndexTypeEnum.summary]: {
-    name: 'kb.Mark Data',
+    name: 'dataset.data.indexes.summary'
-    sourceName: 'kb.Manual Mark'
+  },
+  [DatasetDataIndexTypeEnum.hypothetical]: {
+    name: 'dataset.data.indexes.hypothetical'
+  },
+  [DatasetDataIndexTypeEnum.qa]: {
+    name: 'dataset.data.indexes.qa'
+  },
+  [DatasetDataIndexTypeEnum.custom]: {
+    name: 'dataset.data.indexes.custom'
   }
 };
-export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark];
+
+export enum TrainingModeEnum {
+  'chunk' = 'chunk',
+  'qa' = 'qa'
+  // 'hypothetical' = 'hypothetical',
+  // 'summary' = 'summary',
+  // 'multipleIndex' = 'multipleIndex'
+}
+export const TrainingTypeMap = {
+  [TrainingModeEnum.chunk]: {
+    name: 'chunk'
+  },
+  [TrainingModeEnum.qa]: {
+    name: 'qa'
+  }
+  // [TrainingModeEnum.hypothetical]: {
+  //   name: 'hypothetical'
+  // },
+  // [TrainingModeEnum.summary]: {
+  //   name: 'summary'
+  // },
+  // [TrainingModeEnum.multipleIndex]: {
+  //   name: 'multipleIndex'
+  // }
+};
 
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/dataset/controller.d.ts

@@ -0,0 +1,27 @@
+import type { DatasetDataIndexItemType, DatasetDataSchemaType } from './type';
+
+export type CreateDatasetDataProps = {
+  teamId: string;
+  tmbId: string;
+  datasetId: string;
+  collectionId: string;
+  q: string;
+  a?: string;
+  indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
+};
+
+export type UpdateDatasetDataProps = {
+  dataId: string;
+  q?: string;
+  a?: string;
+  indexes?: (Omit<DatasetDataIndexItemType, 'dataId'> & {
+    dataId?: string; // pg data id
+  })[];
+};
+
+export type PatchIndexesProps = {
+  type: 'create' | 'update' | 'delete';
+  index: Omit<DatasetDataIndexItemType, 'dataId'> & {
+    dataId?: string;
+  };
+};

packages/global/core/dataset/type.d.ts

@@ -1,6 +1,14 @@
+import type { VectorModelItemType } from '../../core/ai/model.d';
 import { PermissionTypeEnum } from '../../support/permission/constant';
-import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant';
+import { PushDatasetDataChunkProps } from './api';
+import {
+  DatasetCollectionTypeEnum,
+  DatasetDataIndexTypeEnum,
+  DatasetTypeEnum,
+  TrainingModeEnum
+} from './constant';
 
+/* schema */
 export type DatasetSchemaType = {
   _id: string;
   parentId: string;
@@ -33,13 +41,33 @@ export type DatasetCollectionSchemaType = {
   };
 };
 
+export type DatasetDataIndexItemType = {
+  defaultIndex: boolean;
+  dataId: string; // pg data id
+  type: `${DatasetDataIndexTypeEnum}`;
+  text: string;
+};
+export type DatasetDataSchemaType = {
+  _id: string;
+  userId: string;
+  teamId: string;
+  tmbId: string;
+  datasetId: string;
+  collectionId: string;
+  datasetId: string;
+  collectionId: string;
+  q: string; // large chunks or question
+  a: string; // answer or custom content
+  indexes: DatasetDataIndexItemType[];
+};
+
 export type DatasetTrainingSchemaType = {
   _id: string;
   userId: string;
   teamId: string;
   tmbId: string;
   datasetId: string;
-  datasetCollectionId: string;
+  collectionId: string;
   billId: string;
   expireAt: Date;
   lockTime: Date;
@@ -48,6 +76,7 @@ export type DatasetTrainingSchemaType = {
   prompt: string;
   q: string;
   a: string;
+  indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 
 export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datasetId'> & {
@@ -55,41 +84,31 @@ export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datas
 };
 
 /* ================= dataset ===================== */
-
+export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel'> & {
-/* ================= collection ===================== */
+  vectorModel: VectorModelItemType;
-export type DatasetCollectionItemType = DatasetCollectionSchemaType & {
+  isOwner: boolean;
   canWrite: boolean;
 };
 
+/* ================= collection ===================== */
+export type DatasetCollectionItemType = CollectionWithDatasetType & {
+  canWrite: boolean;
+  sourceName: string;
+  sourceId?: string;
+};
+
 /* ================= data ===================== */
-export type PgRawDataItemType = {
+export type DatasetDataItemType = {
-  id: string;
-  q: string;
-  a: string;
-  team_id: string;
-  tmb_id: string;
-  dataset_id: string;
-  collection_id: string;
-};
-export type PgDataItemType = {
-  id: string;
-  q: string;
-  a: string;
-  teamId: string;
-  tmbId: string;
-  datasetId: string;
-  collectionId: string;
-};
-export type DatasetChunkItemType = {
-  q: string;
-  a: string;
-};
-export type DatasetDataItemType = DatasetChunkItemType & {
   id: string;
   datasetId: string;
   collectionId: string;
   sourceName: string;
   sourceId?: string;
+  q: string;
+  a: string;
+  indexes: DatasetDataIndexItemType[];
+  isOwner: boolean;
+  canWrite: boolean;
 };
 
 /* --------------- file ---------------------- */
@@ -109,9 +128,6 @@ export type DatasetFileSchema = {
 };
 
 /* ============= search =============== */
-export type SearchDataResultItemType = PgRawDataItemType & {
-  score: number;
-};
 export type SearchDataResponseItemType = DatasetDataItemType & {
   score: number;
 };

packages/global/core/dataset/utils.ts

@@ -1,4 +1,4 @@
-import { DatasetCollectionTypeEnum } from './constant';
+import { DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant';
 import { getFileIcon } from '../../common/file/icon';
 import { strIsLink } from '../../common/string/tools';
 
@@ -44,3 +44,14 @@ export function getSourceNameIcon({
   }
   return '/imgs/files/collection.svg';
 }
+
+export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: string }) {
+  const { q = '', a, dataId } = props || {};
+  const qaStr = `${q}\n${a}`.trim();
+  return {
+    defaultIndex: true,
+    type: a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
+    text: a ? qaStr : q,
+    dataId
+  };
+}

packages/global/core/module/api.d.ts

@@ -0,0 +1,3 @@
+import { VectorModelItemType } from '../ai/model.d';
+
+export type SelectedDatasetType = { datasetId: string; vectorModel: VectorModelItemType }[];

packages/global/package.json

@@ -6,7 +6,8 @@
     "timezones-list": "^3.0.2",
     "dayjs": "^1.11.7",
     "encoding": "^0.1.13",
-    "openai": "^4.16.1"
+    "openai": "^4.16.1",
+    "js-tiktoken": "^1.0.7"
   },
   "devDependencies": {
     "@types/node": "^20.8.5"

packages/global/support/user/team/constant.ts

@@ -24,7 +24,8 @@ export const TeamMemberRoleMap = {
 export enum TeamMemberStatusEnum {
   waiting = 'waiting',
   active = 'active',
-  reject = 'reject'
+  reject = 'reject',
+  leave = 'leave'
 }
 export const TeamMemberStatusMap = {
   [TeamMemberStatusEnum.waiting]: {
@@ -38,5 +39,10 @@ export const TeamMemberStatusMap = {
   [TeamMemberStatusEnum.reject]: {
     label: 'user.team.member.reject',
     color: 'red.600'
+  },
+  [TeamMemberStatusEnum.leave]: {
+    label: 'user.team.member.leave',
+    color: 'red.600'
   }
 };
+export const leaveStatus = { $ne: TeamMemberStatusEnum.leave };

packages/global/support/user/team/controller.d.ts

@@ -37,4 +37,7 @@ export type UpdateInviteProps = {
   tmbId: string;
   status: TeamMemberSchema['status'];
 };
-export type InviteMemberResponse = Record<'invite' | 'inValid' | 'inTeam', string[]>;
+export type InviteMemberResponse = Record<
+  'invite' | 'inValid' | 'inTeam',
+  { username: string; userId: string }[]
+>;

packages/global/support/user/team/type.d.ts

@@ -16,6 +16,7 @@ export type TeamMemberSchema = {
   teamId: string;
   userId: string;
   createTime: Date;
+  name: string;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
   defaultTeam: boolean;
@@ -25,6 +26,7 @@ export type TeamItemType = {
   userId: string;
   teamId: string;
   teamName: string;
+  memberName: string;
   avatar: string;
   balance: number;
   tmbId: string;
@@ -39,7 +41,7 @@ export type TeamMemberItemType = {
   userId: string;
   tmbId: string;
   teamId: string;
-  memberUsername: string;
+  memberName: string;
   avatar: string;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;

packages/global/support/wallet/bill/type.d.ts

@@ -15,7 +15,7 @@ export type BillSchema = CreateBillProps & {
 
 export type BillItemType = {
   id: string;
-  username: string;
+  memberName: string;
   time: Date;
   appName: string;
   source: BillSchema['source'];

packages/service/common/api/plusRequest.ts

@@ -15,9 +15,6 @@ interface ResponseDataType {
  * 请求开始
  */
 function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
-  if (config.headers) {
-    config.headers.rootkey = process.env.ROOT_KEY;
-  }
   return config;
 }
 
@@ -62,7 +59,8 @@ const instance = axios.create({
   timeout: 60000, // 超时时间
   headers: {
     'content-type': 'application/json',
-    'Cache-Control': 'no-cache'
+    'Cache-Control': 'no-cache',
+    rootkey: process.env.ROOT_KEY
   }
 });
 

packages/service/common/pg/index.ts

@@ -171,8 +171,7 @@ export async function initPg() {
           tmb_id VARCHAR(50) NOT NULL,
           dataset_id VARCHAR(50) NOT NULL,
           collection_id VARCHAR(50) NOT NULL,
-          q TEXT NOT NULL,
+          data_id VARCHAR(50) NOT NULL
-          a TEXT
       );
       CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64);
     `);

packages/service/core/chat/utils.ts

@@ -1,22 +1,7 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
-import type { NextApiResponse } from 'next';
+import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
-import { countMessagesTokens, countPromptTokens } from '@/global/common/tiktoken';
+import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
-import { adaptRole_Chat2Message } from '@/utils/common/adapt/message';
-
-export type ChatCompletionResponseType = {
-  streamResponse: any;
-  responseMessages: ChatItemType[];
-  responseText: string;
-  totalTokens: number;
-};
-export type StreamResponseType = {
-  chatResponse: any;
-  messages: ChatItemType[];
-  res: NextApiResponse;
-  model: string;
-  [key: string]: any;
-};
 
 /* slice chat context by tokens */
 export function ChatContextFilter({

packages/service/core/dataset/collection/schema.ts

@@ -56,8 +56,7 @@ const DatasetCollectionSchema = new Schema({
         ref: 'dataset.files'
       },
       rawLink: {
-        type: String,
+        type: String
-        default: ''
       },
       // 451 初始化
       pgCollectionId: {

packages/service/core/dataset/controller.ts

@@ -1,5 +1,25 @@
 import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
+import { MongoDataset } from './schema';
+
+/* ============= dataset ========== */
+/* find all datasetId by top datasetId */
+export async function findDatasetIdTreeByTopDatasetId(
+  id: string,
+  result: string[] = []
+): Promise<string[]> {
+  let allChildrenIds = [...result];
+
+  // find children
+  const children = await MongoDataset.find({ parentId: id });
+
+  for (const child of children) {
+    const grandChildrenIds = await findDatasetIdTreeByTopDatasetId(child._id, result);
+    allChildrenIds = allChildrenIds.concat(grandChildrenIds);
+  }
+
+  return [String(id), ...allChildrenIds];
+}
 
 export async function getCollectionWithDataset(collectionId: string) {
   const data = (

packages/service/core/dataset/data/schema.ts

@@ -0,0 +1,78 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import {
+  TeamCollectionName,
+  TeamMemberCollectionName
+} from '@fastgpt/global/support/user/team/constant';
+import { DatasetCollectionName } from '../schema';
+import { DatasetColCollectionName } from '../collection/schema';
+import { DatasetDataIndexTypeMap } from '@fastgpt/global/core/dataset/constant';
+
+export const DatasetDataCollectionName = 'dataset.datas';
+
+const DatasetDataSchema = new Schema({
+  teamId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamCollectionName,
+    required: true
+  },
+  tmbId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamMemberCollectionName,
+    required: true
+  },
+  datasetId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetCollectionName,
+    required: true
+  },
+  collectionId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetColCollectionName,
+    required: true
+  },
+  q: {
+    type: String,
+    required: true
+  },
+  a: {
+    type: String,
+    default: ''
+  },
+  indexes: {
+    type: [
+      {
+        defaultIndex: {
+          type: Boolean,
+          default: false
+        },
+        type: {
+          type: String,
+          enum: Object.keys(DatasetDataIndexTypeMap),
+          required: true
+        },
+        dataId: {
+          type: String,
+          required: true
+        },
+        text: {
+          type: String,
+          required: true
+        }
+      }
+    ],
+    default: []
+  }
+});
+
+try {
+  DatasetDataSchema.index({ userId: 1 });
+  DatasetDataSchema.index({ datasetId: 1 });
+  DatasetDataSchema.index({ collectionId: 1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDatasetData: Model<DatasetDataSchemaType> =
+  models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);

packages/service/core/dataset/training/schema.ts

@@ -2,7 +2,7 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
-import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
+import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
 import { DatasetColCollectionName } from '../collection/schema';
 import { DatasetCollectionName } from '../schema';
 import {
@@ -33,12 +33,13 @@ const TrainingDataSchema = new Schema({
     ref: DatasetCollectionName,
     required: true
   },
-  datasetCollectionId: {
+  collectionId: {
     type: Schema.Types.ObjectId,
     ref: DatasetColCollectionName,
     required: true
   },
   billId: {
+    // concat bill
     type: String,
     default: ''
   },
@@ -48,6 +49,7 @@ const TrainingDataSchema = new Schema({
     required: true
   },
   expireAt: {
+    // It will be deleted after 7 days
     type: Date,
     default: () => new Date()
   },
@@ -56,6 +58,7 @@ const TrainingDataSchema = new Schema({
     default: () => new Date('2000/1/1')
   },
   model: {
+    // ai model
     type: String,
     required: true
   },
@@ -71,13 +74,29 @@ const TrainingDataSchema = new Schema({
   a: {
     type: String,
     default: ''
+  },
+  indexes: {
+    type: [
+      {
+        type: {
+          type: String,
+          enum: Object.keys(DatasetDataIndexTypeMap),
+          required: true
+        },
+        text: {
+          type: String,
+          required: true
+        }
+      }
+    ],
+    default: []
   }
 });
 
 try {
   TrainingDataSchema.index({ lockTime: 1 });
   TrainingDataSchema.index({ userId: 1 });
-  TrainingDataSchema.index({ datasetCollectionId: 1 });
+  TrainingDataSchema.index({ collectionId: 1 });
   TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 });
 } catch (error) {
   console.log(error);

packages/service/support/activity/promotion/schema.ts

@@ -23,6 +23,7 @@ const PromotionRecordSchema = new Schema({
     enum: ['pay', 'register']
   },
   amount: {
+    // 1 * PRICE_SCALE
     type: Number,
     required: true
   }

packages/service/support/outLink/tools.ts

@@ -30,7 +30,7 @@ export const pushResult2Remote = async ({
   shareId?: string;
   responseData?: any[];
 }) => {
-  if (!shareId || !authToken) return;
+  if (!shareId || !authToken || !global.systemEnv.pluginBaseUrl) return;
   try {
     const outLink = await MongoOutLink.findOne({
       shareId

packages/service/support/permission/auth/common.ts

@@ -1,5 +1,7 @@
+import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { parseHeaderCert } from '../controller';
 import { AuthModeType } from '../type';
+import { authOutLinkValid } from './outLink';
 
 export const authCert = async (props: AuthModeType) => {
   const result = await parseHeaderCert(props);
@@ -10,3 +12,22 @@ export const authCert = async (props: AuthModeType) => {
     canWrite: true
   };
 };
+export async function authCertAndShareId({
+  shareId,
+  ...props
+}: AuthModeType & { shareId?: string }) {
+  if (!shareId) {
+    return authCert(props);
+  }
+
+  const { app } = await authOutLinkValid({ shareId });
+
+  return {
+    teamId: String(app.teamId),
+    tmbId: String(app.tmbId),
+    authType: AuthUserTypeEnum.outLink,
+    apikey: '',
+    isOwner: false,
+    canWrite: false
+  };
+}

packages/service/support/permission/auth/dataset.ts

@@ -27,11 +27,11 @@ export async function authDataset({
   }
 > {
   const result = await parseHeaderCert(props);
-  const { userId, teamId, tmbId } = result;
+  const { teamId, tmbId } = result;
   const { role } = await getTeamInfoByTmbId({ tmbId });
 
   const { dataset, isOwner, canWrite } = await (async () => {
-    const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toJSON();
+    const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject();
 
     if (!dataset) {
       return Promise.reject(DatasetErrEnum.unAuthDataset);

packages/service/support/permission/auth/openapi.ts

@@ -6,7 +6,6 @@ import { getTeamInfoByTmbId } from '../../user/team/controller';
 import { MongoOpenApi } from '../../openapi/schema';
 import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
-import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
 
 export async function authOpenApiKeyCrud({
   id,

packages/service/support/permission/auth/user.ts

@@ -37,13 +37,11 @@ export async function authUserRole(props: AuthModeType): Promise<
     teamOwner: boolean;
   }
 > {
-  const { userId, teamId, tmbId } = await parseHeaderCert(props);
+  const result = await parseHeaderCert(props);
-  const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId });
+  const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId: result.tmbId });
 
   return {
-    userId,
+    ...result,
-    teamId,
-    tmbId,
     isOwner: true,
     role: userRole,
     teamOwner: userRole === TeamMemberRoleEnum.owner,

packages/service/support/user/team/controller.ts

@@ -4,57 +4,42 @@ import {
   TeamMemberRoleEnum,
   TeamMemberStatusEnum,
   TeamCollectionName,
-  TeamMemberCollectionName
+  TeamMemberCollectionName,
+  leaveStatus
 } from '@fastgpt/global/support/user/team/constant';
 
-export async function getTeamInfoByTmbId({
+async function getTeam(match: Record<string, any>): Promise<TeamItemType> {
-  tmbId,
-  userId
-}: {
-  tmbId?: string;
-  userId?: string;
-}): Promise<TeamItemType> {
-  if (!tmbId && !userId) {
-    return Promise.reject('tmbId or userId is required');
-  }
-
   const db = connectionMongo?.connection?.db;
 
   const TeamMember = db.collection(TeamMemberCollectionName);
 
   const results = await TeamMember.aggregate([
     {
-      $match: tmbId
+      $match: match
-        ? {
-            _id: new Types.ObjectId(tmbId)
-          }
-        : {
-            userId: new Types.ObjectId(userId),
-            defaultTeam: true
-          }
     },
     {
       $lookup: {
-        from: TeamCollectionName, // 关联的集合名
+        from: TeamCollectionName,
-        localField: 'teamId', // TeamMember 集合中用于关联的字段
+        localField: 'teamId',
-        foreignField: '_id', // Team 集合中用于关联的字段
+        foreignField: '_id',
-        as: 'team' // 查询结果中的字段名，存放关联查询的结果
+        as: 'team'
       }
     },
     {
-      $unwind: '$team' // 将查询结果中的 team 字段展开，变成一个对象
+      $unwind: '$team'
     }
   ]).toArray();
   const tmb = results[0];
 
   if (!tmb) {
-    return Promise.reject('team not exist');
+    return Promise.reject('member not exist');
   }
 
   return {
     userId: String(tmb.userId),
     teamId: String(tmb.teamId),
     teamName: tmb.team.name,
+    memberName: tmb.name,
     avatar: tmb.team.avatar,
     balance: tmb.team.balance,
     tmbId: String(tmb._id),
@@ -65,11 +50,31 @@ export async function getTeamInfoByTmbId({
     maxSize: tmb.team.maxSize
   };
 }
+
+export async function getTeamInfoByTmbId({ tmbId }: { tmbId: string }) {
+  if (!tmbId) {
+    return Promise.reject('tmbId or userId is required');
+  }
+  return getTeam({
+    _id: new Types.ObjectId(tmbId),
+    status: leaveStatus
+  });
+}
+
+export async function getUserDefaultTeam({ userId }: { userId: string }) {
+  if (!userId) {
+    return Promise.reject('tmbId or userId is required');
+  }
+  return getTeam({
+    userId: new Types.ObjectId(userId),
+    defaultTeam: true
+  });
+}
 export async function createDefaultTeam({
   userId,
   teamName = 'My Team',
   avatar = '/icon/logo.svg',
-  balance = 0,
+  balance,
   maxSize = 5
 }: {
   userId: string;
@@ -103,6 +108,7 @@ export async function createDefaultTeam({
     await TeamMember.insertOne({
       teamId: insertedId,
       userId,
+      name: 'Owner',
       role: TeamMemberRoleEnum.owner,
       status: TeamMemberStatusEnum.active,
       createTime: new Date(),
@@ -116,7 +122,7 @@ export async function createDefaultTeam({
       },
       {
         $set: {
-          balance,
+          ...(balance !== undefined && { balance }),
           maxSize
         }
       }

packages/service/support/wallet/bill/schema.ts

@@ -36,6 +36,7 @@ const BillSchema = new Schema({
     default: () => new Date()
   },
   total: {
+    // 1 * PRICE_SCALE
     type: Number,
     required: true
   },

pnpm-lock.yaml

@@ -51,6 +51,9 @@ importers:
       encoding:
         specifier: ^0.1.13
         version: registry.npmmirror.com/encoding@0.1.13
+      js-tiktoken:
+        specifier: ^1.0.7
+        version: registry.npmmirror.com/js-tiktoken@1.0.7
       openai:
         specifier: ^4.16.1
         version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13)
@@ -200,12 +203,6 @@ importers:
       immer:
         specifier: ^9.0.19
         version: registry.npmmirror.com/immer@9.0.21
-      js-cookie:
-        specifier: ^3.0.5
-        version: registry.npmmirror.com/js-cookie@3.0.5
-      js-tiktoken:
-        specifier: ^1.0.7
-        version: registry.npmmirror.com/js-tiktoken@1.0.7
       jschardet:
         specifier: ^3.0.0
         version: registry.npmmirror.com/jschardet@3.0.0
@@ -242,9 +239,6 @@ importers:
       papaparse:
         specifier: ^5.4.1
         version: registry.npmmirror.com/papaparse@5.4.1
-      pg-query-stream:
-        specifier: ^4.5.3
-        version: registry.npmmirror.com/pg-query-stream@4.5.3(pg@8.11.3)
       react:
         specifier: 18.2.0
         version: registry.npmmirror.com/react@18.2.0
@@ -8721,13 +8715,6 @@ packages:
       set-function-name: registry.npmmirror.com/set-function-name@2.0.1
     dev: true
 
-  registry.npmmirror.com/js-cookie@3.0.5:
-    resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-cookie/-/js-cookie-3.0.5.tgz}
-    name: js-cookie
-    version: 3.0.5
-    engines: {node: '>=14'}
-    dev: false
-
   registry.npmmirror.com/js-sdsl@4.4.2:
     resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz}
     name: js-sdsl
@@ -10629,17 +10616,6 @@ packages:
     version: 2.6.2
     dev: false
 
-  registry.npmmirror.com/pg-cursor@2.10.3(pg@8.11.3):
-    resolution: {integrity: sha512-rDyBVoqPVnx/PTmnwQAYgusSeAKlTL++gmpf5klVK+mYMFEqsOc6VHHZnPKc/4lOvr4r6fiMuoxSFuBF1dx4FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-cursor/-/pg-cursor-2.10.3.tgz}
-    id: registry.npmmirror.com/pg-cursor/2.10.3
-    name: pg-cursor
-    version: 2.10.3
-    peerDependencies:
-      pg: ^8
-    dependencies:
-      pg: registry.npmmirror.com/pg@8.11.3
-    dev: false
-
   registry.npmmirror.com/pg-int8@1.0.1:
     resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-int8/-/pg-int8-1.0.1.tgz}
     name: pg-int8
@@ -10669,18 +10645,6 @@ packages:
     name: pg-protocol
     version: 1.6.0
 
-  registry.npmmirror.com/pg-query-stream@4.5.3(pg@8.11.3):
-    resolution: {integrity: sha512-ufa94r/lHJdjAm3+zPZEO0gXAmCb4tZPaOt7O76mjcxdL/HxwTuryy76km+u0odBBgtfdKFYq/9XGfiYeQF0yA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-query-stream/-/pg-query-stream-4.5.3.tgz}
-    id: registry.npmmirror.com/pg-query-stream/4.5.3
-    name: pg-query-stream
-    version: 4.5.3
-    peerDependencies:
-      pg: ^8
-    dependencies:
-      pg: registry.npmmirror.com/pg@8.11.3
-      pg-cursor: registry.npmmirror.com/pg-cursor@2.10.3(pg@8.11.3)
-    dev: false
-
   registry.npmmirror.com/pg-types@2.2.0:
     resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-types/-/pg-types-2.2.0.tgz}
     name: pg-types

projects/app/next-i18next.config.js

@@ -8,5 +8,8 @@ module.exports = {
     defaultLocale: 'zh',
     locales: ['en', 'zh', 'zh-Hans', 'zh-CN'],
     localeDetection: false
-  }
+  },
+  localePath:
+    typeof window === 'undefined' ? require('path').resolve('./public/locales') : '/locales',
+  reloadOnPrerender: process.env.NODE_ENV === 'development'
 };

projects/app/package.json

@@ -33,8 +33,6 @@
     "hyperdown": "^2.4.29",
     "i18next": "^22.5.1",
     "immer": "^9.0.19",
-    "js-cookie": "^3.0.5",
-    "js-tiktoken": "^1.0.7",
     "jschardet": "^3.0.0",
     "jsdom": "^22.1.0",
     "jsonwebtoken": "^9.0.2",
@@ -47,7 +45,6 @@
     "next-i18next": "^13.3.0",
     "nprogress": "^0.2.0",
     "papaparse": "^5.4.1",
-    "pg-query-stream": "^4.5.3",
     "react": "18.2.0",
     "react-day-picker": "^8.7.1",
     "react-dom": "18.2.0",

projects/app/public/docs/versionIntro.md

@@ -1,9 +1,12 @@
 ### Fast GPT V4.6
 
-1. 新增 - 团队空间试用版。
+1. 新增 - 团队空间
-2. 新增 - OpenAI语音播报。
+2. 新增 - 多路向量（多个向量映射一组数据）
-3. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
+3. 新增 - tts语音
-4. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
+4. 线上环境新增 - ReRank向量召回，提高召回精度
-5. [使用文档](https://doc.fastgpt.in/docs/intro/)
+5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
-6. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+6. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
-7. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
+7. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
+8. [使用文档](https://doc.fastgpt.in/docs/intro/)
+9. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+10. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)

projects/app/public/locales/en/common.json

@@ -292,17 +292,32 @@
       "Select One Collection To Store": "Select the collection to store"
     },
     "data": {
+      "Add Index": "Add Index",
       "Can not delete tip": "No modification permission",
       "Can not edit": "No edit permission",
+      "Custom Index Number": "Custom index{{number}}",
+      "Default Index": "Default Index",
+      "Delete Success Tip": "",
       "Delete Tip": "Confirm to delete the data?",
       "File import": "File Import",
+      "Index Edit": "Data Index",
+      "Index Placeholder": "Enter the index text content",
       "Input Data": "Import Data",
       "Input Success Tip": "Succeeded in importing data",
       "Update Data": "Update Data",
-      "Update Success Tip": "Update data successfully"
+      "Update Success Tip": "Update data successfully",
+      "edit": {
+        "Content": "Content",
+        "Course": "Document",
+        "Delete": "Delete",
+        "Index": "Index({{amount}})"
+      }
     },
     "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
-    "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!"
+    "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!",
+    "test": {
+      "noResult": "Search results are empty"
+    }
   },
   "error": {
     "team": {
@@ -461,9 +476,11 @@
     "Bill Detail": "Bill Detail",
     "Change": "Change",
     "Copy invite url": "Copy invitation link",
+    "Edit name": "Click to modify nickname",
     "Invite Url": "Invite Url",
     "Invite url tip": "Friends who register through this link will be permanently bound to you, and you will get a certain balance reward when they recharge. In addition, when friends register with their mobile phone number, you will get 5 yuan reward immediately.",
     "Language": "Language",
+    "Member Name": "Name",
     "Notice": "Notice",
     "Old password is error": "Old password is error",
     "OpenAI Account Setting": "OpenAI Account Setting",
@@ -513,12 +530,15 @@
       "Leave Team Failed": "Leave Team Failed",
       "Manage": "Team Manage",
       "Member": "Member",
+      "Member Name": "Member",
       "Over Max Member Tip": "Team max {{max}} people",
       "Personal Team": "Personal",
       "Processing invitations": "Processing invitations",
       "Processing invitations Tips": "You have {{amount}} of team invitations that need to be processed",
       "Reinvite": "Reinvite",
-      "Remove Member Confirm Tip": "Confirm to move {{username}} off the team",
+      "Remove Member Confirm Tip": "Are you sure you want to move {{username}} off the team? All its resources are transferred to the team creator's account.",
+      "Remove Member Failed": "Removing a team member failed",
+      "Remove Member Success": "Removing a team member succeeded",
       "Remove Member Tip": "Move out team",
       "Role": "Role",
       "Select Team": "Select Team",

projects/app/public/locales/zh/common.json

@@ -292,17 +292,32 @@
       "Select One Collection To Store": "选择一个文件进行存储"
     },
     "data": {
+      "Add Index": "新增自定义索引",
       "Can not delete tip": "无修改权限",
       "Can not edit": "无编辑权限",
+      "Custom Index Number": "自定义索引{{number}}",
+      "Default Index": "默认索引",
+      "Delete Success Tip": "删除成功",
       "Delete Tip": "确认删除该条数据？",
       "File import": "文件导入",
+      "Index Edit": "数据索引",
+      "Index Placeholder": "输入索引文本内容",
       "Input Data": "导入新数据",
       "Input Success Tip": "导入数据成功",
       "Update Data": "更新数据",
-      "Update Success Tip": "更新数据成功"
+      "Update Success Tip": "更新数据成功",
+      "edit": {
+        "Content": "数据内容",
+        "Course": "说明文档",
+        "Delete": "删除数据",
+        "Index": "数据索引({{amount}})"
+      }
     },
     "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
-    "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！"
+    "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！",
+    "test": {
+      "noResult": "搜索结果为空"
+    }
   },
   "error": {
     "team": {
@@ -461,9 +476,11 @@
     "Bill Detail": "账单详情",
     "Change": "变更",
     "Copy invite url": "复制邀请链接",
+    "Edit name": "点击修改昵称",
     "Invite Url": "邀请链接",
     "Invite url tip": "通过该链接注册的好友将永久与你绑定，其充值时你会获得一定余额奖励。\n此外，好友使用手机号注册时，你将立即获得 5 元奖励。\n奖励会发送到您的默认团队中。",
     "Language": "语言",
+    "Member Name": "昵称",
     "Notice": "通知",
     "Old password is error": "旧密码错误",
     "OpenAI Account Setting": "OpenAI 账号配置",
@@ -513,12 +530,15 @@
       "Leave Team Failed": "离开团队异常",
       "Manage": "团队管理",
       "Member": "成员",
+      "Member Name": "成员名",
       "Over Max Member Tip": "团队最多{{max}}人",
       "Personal Team": "个人团队",
       "Processing invitations": "处理邀请",
       "Processing invitations Tips": "你有{{amount}}个需要处理的团队邀请",
       "Reinvite": "重新邀请",
-      "Remove Member Confirm Tip": "确认将 {{username}} 移出团队？",
+      "Remove Member Confirm Tip": "确认将 {{username}} 移出团队？其所有资源将转让到团队创建者的账户内。",
+      "Remove Member Failed": "移除团队成员异常",
+      "Remove Member Success": "移除团队成员成功",
       "Remove Member Tip": "移出团队",
       "Role": "身份",
       "Select Team": "团队选择",

projects/app/src/components/ChatBox/QuoteModal.tsx

@@ -10,13 +10,12 @@ import InputDataModal, {
   type InputDataType
 } from '@/pages/dataset/detail/components/InputDataModal';
 import MyModal from '../MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useRouter } from 'next/router';
 import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import MyTooltip from '../MyTooltip';
 import NextLink from 'next/link';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import { useUserStore } from '@/web/support/user/useUserStore';
 
 const QuoteModal = ({
   rawSearch = [],
@@ -29,10 +28,9 @@ const QuoteModal = ({
   const { isPc } = useSystemStore();
   const theme = useTheme();
   const router = useRouter();
-  const { userInfo } = useUserStore();
   const { toast } = useToast();
   const { setIsLoading, Loading } = useLoading();
-  const [editInputData, setEditInputData] = useState<InputDataType & { datasetId: string }>();
+  const [editInputData, setEditInputData] = useState<InputDataType & { collectionId: string }>();
 
   const isShare = useMemo(() => router.pathname === '/chat/share', [router.pathname]);
 
@@ -99,7 +97,7 @@ const QuoteModal = ({
                   color={'black'}
                   sourceName={item.sourceName}
                   sourceId={item.sourceId}
-                  addr={!isShare}
+                  canView={!isShare}
                 />
                 <Box flex={1} />
                 {!isShare && (
@@ -131,7 +129,7 @@ const QuoteModal = ({
                   <MyTooltip label={t('core.dataset.Quote Length')}>
                     <Flex alignItems={'center'}>
                       <MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
-                      {item.q.length + item.a.length}
+                      {item.q.length + (item.a?.length || 0)}
                     </Flex>
                   </MyTooltip>
                   {!isShare && item.score && (
@@ -183,7 +181,6 @@ const QuoteModal = ({
       </MyModal>
       {editInputData && editInputData.id && (
         <InputDataModal
-          canWrite={userInfo?.team?.canWrite || false}
           onClose={() => setEditInputData(undefined)}
           onSuccess={() => {
             console.log('更新引用成功');
@@ -191,8 +188,8 @@ const QuoteModal = ({
           onDelete={() => {
             console.log('删除引用成功');
           }}
-          datasetId={editInputData.datasetId}
+          defaultValue={editInputData}
-          defaultValues={editInputData}
+          collectionId={editInputData.collectionId}
         />
       )}
     </>

projects/app/src/components/ChatBox/ResponseTags.tsx

@@ -2,7 +2,7 @@ import React, { useMemo, useState } from 'react';
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d';
 import type { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { Flex, BoxProps, useDisclosure, Image, useTheme } from '@chakra-ui/react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import dynamic from 'next/dynamic';

projects/app/src/components/ChatBox/SelectMarkCollection.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
 import Avatar from '../Avatar';
@@ -8,6 +8,7 @@ import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/
 import dynamic from 'next/dynamic';
 import { AdminFbkType } from '@fastgpt/global/core/chat/type.d';
 import SelectCollections from '@/web/core/dataset/components/SelectCollections';
+import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
 
 const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal'));
 
@@ -163,15 +164,13 @@ const SelectMarkCollection = ({
       {adminMarkData.datasetId && adminMarkData.collectionId && (
         <InputDataModal
           onClose={onClose}
-          datasetId={adminMarkData.datasetId}
+          collectionId={adminMarkData.collectionId}
-          defaultValues={{
+          defaultValue={{
             id: adminMarkData.dataId,
-            collectionId: adminMarkData.collectionId,
-            sourceName: '手动标注',
             q: adminMarkData.q,
-            a: adminMarkData.a
+            a: adminMarkData.a,
+            indexes: [getDefaultIndex({ dataId: `${Date.now()}` })]
           }}
-          canWrite
           onSuccess={(data) => {
             if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) {
               return onClose();

projects/app/src/components/ChatBox/WholeResponseModal.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo, useState } from 'react';
 import { Box, useTheme, Flex, Image } from '@chakra-ui/react';
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { ModuleTemplatesFlat } from '@/constants/flow/ModuleTemplate';
 import Tabs from '../Tabs';
 

projects/app/src/components/ChatBox/index.tsx

@@ -30,7 +30,7 @@ import {
 } from '@chakra-ui/react';
 import { feConfigs } from '@/web/common/system/staticData';
 import { eventBus } from '@/web/common/utils/eventbus';
-import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
+import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt';
 import { useMarkdown } from '@/web/common/hooks/useMarkdown';
 import { ModuleItemType } from '@fastgpt/global/core/module/type.d';
 import { VariableInputEnum } from '@/constants/app';
@@ -41,7 +41,7 @@ import { htmlTemplate } from '@/constants/common';
 import { useRouter } from 'next/router';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { customAlphabet } from 'nanoid';
 import { adminUpdateChatFeedback, userUpdateChatFeedback } from '@/web/core/chat/api';
 import type { AdminMarkType } from './SelectMarkCollection';

projects/app/src/components/CommunityModal/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Button, ModalFooter, ModalBody } from '@chakra-ui/react';
 import MyModal from '../MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import Markdown from '../Markdown';
 
 const md = `

projects/app/src/components/Language/index.tsx

@@ -1,48 +0,0 @@
-import React, { useState } from 'react';
-import { Menu, MenuButton, MenuItem, MenuList, MenuButtonProps } from '@chakra-ui/react';
-import { getLangStore, LangEnum, setLangStore, langMap } from '@/web/common/utils/i18n';
-import MyIcon from '@/components/Icon';
-import { useTranslation } from 'react-i18next';
-import { useRouter } from 'next/router';
-
-const Language = (props: MenuButtonProps) => {
-  const router = useRouter();
-  const { i18n } = useTranslation();
-
-  const [language, setLanguage] = useState<`${LangEnum}`>(getLangStore());
-
-  return (
-    <Menu autoSelect={false}>
-      <MenuButton
-        {...props}
-        sx={{
-          '& span': {
-            flex: 0
-          }
-        }}
-      >
-        <MyIcon name={langMap[language].icon as any} w={['18px', '22px']} />
-      </MenuButton>
-      <MenuList w="max-content" minW="120px">
-        {Object.entries(langMap).map(([key, lang]) => (
-          <MenuItem
-            key={key}
-            display="flex"
-            alignItems="center"
-            onClick={() => {
-              const lang = key as `${LangEnum}`;
-              setLangStore(lang);
-              setLanguage(lang);
-              i18n?.changeLanguage?.(lang);
-              router.reload();
-            }}
-          >
-            {lang.label}
-          </MenuItem>
-        ))}
-      </MenuList>
-    </Menu>
-  );
-};
-
-export default React.memo(Language);

projects/app/src/components/Layout/navbarPhone.tsx

@@ -2,7 +2,7 @@ import React, { useMemo } from 'react';
 import { useRouter } from 'next/router';
 import { Flex, Box } from '@chakra-ui/react';
 import { useChatStore } from '@/web/core/chat/storeChat';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import Badge from '../Badge';
 import MyIcon from '../Icon';
 

projects/app/src/components/Tabs/index.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo } from 'react';
 import { Box, Grid } from '@chakra-ui/react';
 import type { GridProps } from '@chakra-ui/react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 // @ts-ignore
 interface Props extends GridProps {

projects/app/src/components/common/ParentPaths/index.tsx

@@ -2,7 +2,7 @@ import MyIcon from '@/components/Icon';
 import { Box, Flex } from '@chakra-ui/react';
 import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
 import React, { useMemo } from 'react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 const ParentPaths = (props: {
   paths?: ParentTreePathItemType[];

projects/app/src/components/common/Textarea/TagTextarea.tsx

@@ -10,7 +10,7 @@ import {
   useTheme
 } from '@chakra-ui/react';
 import { useToast } from '@/web/common/hooks/useToast';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 type Props = BoxProps & { defaultValues: string[]; onUpdate: (e: string[]) => void };
 

projects/app/src/components/core/dataset/SelectModal.tsx

@@ -2,7 +2,7 @@ import { getDatasets, getDatasetPaths } from '@/web/core/dataset/api';
 import MyModal from '@/components/MyModal';
 import { useQuery } from '@tanstack/react-query';
 import React, { Dispatch, useMemo, useState } from 'react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { Box, Flex, ModalHeader } from '@chakra-ui/react';
 import MyIcon from '@/components/Icon';

projects/app/src/components/core/module/AIChatSettingsModal.tsx

@@ -1,6 +1,6 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { EditFormType } from '@/web/core/app/basicSettings';
 import { useForm } from 'react-hook-form';
 import {

projects/app/src/components/core/module/DatasetSelectModal.tsx

@@ -14,14 +14,14 @@ import {
 import Avatar from '@/components/Avatar';
 import { useForm } from 'react-hook-form';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import type { SelectedDatasetType } from '@/types/core/dataset';
+import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d';
 import { useToast } from '@/web/common/hooks/useToast';
 import MySlider from '@/components/Slider';
 import MyTooltip from '@/components/MyTooltip';
 import MyModal from '@/components/MyModal';
 import MyIcon from '@/components/Icon';
 import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { feConfigs } from '@/web/common/system/staticData';
 import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal';

projects/app/src/components/core/module/Flow/ImportSettings.tsx

@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import { Textarea, Button, ModalBody, ModalFooter } from '@chakra-ui/react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useFlowProviderStore } from './FlowProvider';
 

projects/app/src/components/core/module/Flow/SelectAppModal.tsx

@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal';
 import { useQuery } from '@tanstack/react-query';
 import type { SelectAppItemType } from '@fastgpt/global/core/module/type';
 import Avatar from '@/components/Avatar';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { useAppStore } from '@/web/core/app/store/useAppStore';

projects/app/src/components/core/module/Flow/TemplateList.tsx

@@ -10,7 +10,7 @@ import Avatar from '@/components/Avatar';
 import { useFlowProviderStore } from './FlowProvider';
 import { customAlphabet } from 'nanoid';
 import { appModule2FlowNode } from '@/utils/adapt';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useRouter } from 'next/router';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6);
 import MyIcon from '@/components/Icon';

projects/app/src/components/core/module/Flow/components/modules/FieldEditModal.tsx

@@ -13,7 +13,7 @@ import { useForm } from 'react-hook-form';
 import MyModal from '@/components/MyModal';
 import Avatar from '@/components/Avatar';
 import { FlowNodeValTypeEnum } from '@fastgpt/global/core/module/node/constant';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import MySelect from '@/components/Select';
 
 const typeSelectList = [

projects/app/src/components/core/module/Flow/components/modules/NodeCard.tsx

@@ -5,7 +5,7 @@ import Avatar from '@/components/Avatar';
 import type { FlowModuleItemType } from '@fastgpt/global/core/module/type.d';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useEditTitle } from '@/web/common/hooks/useEditTitle';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useFlowProviderStore, onChangeNode } from '../../FlowProvider';

projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx

@@ -15,7 +15,7 @@ import {
   FlowNodeValTypeEnum,
   FlowNodeSpecialInputKeyEnum
 } from '@fastgpt/global/core/module/node/constant';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import SourceHandle from '../render/SourceHandle';
 import MyTooltip from '@/components/MyTooltip';
 import { onChangeNode } from '../../FlowProvider';

projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx

@@ -26,12 +26,12 @@ import MySlider from '@/components/Slider';
 import MyTooltip from '@/components/MyTooltip';
 import TargetHandle from './TargetHandle';
 import MyIcon from '@/components/Icon';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { AIChatProps } from '@/types/core/aiChat';
 import { chatModelList } from '@/web/common/system/staticData';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { SelectedDatasetType } from '@/types/core/dataset';
+import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d';
 import { useQuery } from '@tanstack/react-query';
 import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import type { EditFieldModeType, EditFieldType } from '../modules/FieldEditModal';

projects/app/src/components/support/apikey/Table.tsx

@@ -33,7 +33,7 @@ import dayjs from 'dayjs';
 import { AddIcon, QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
 import { feConfigs } from '@/web/common/system/staticData';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import MyIcon from '@/components/Icon';
 import MyModal from '@/components/MyModal';
 import { useForm } from 'react-hook-form';

projects/app/src/components/support/permission/IconText/index.tsx

@@ -2,7 +2,7 @@ import React from 'react';
 import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
 import { Box, Flex, FlexProps } from '@chakra-ui/react';
 import MyIcon from '@/components/Icon';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 const PermissionIconText = ({
   permission,

projects/app/src/components/support/permission/Radio/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import MyRadio from '@/components/Radio';
 import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 const PermissionRadio = ({
   value,

projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx

@@ -1,6 +1,6 @@
 import React, { useCallback, useState } from 'react';
 import { useForm } from 'react-hook-form';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { compressImgAndUpload } from '@/web/common/file/controller';
 import { useToast } from '@/web/common/hooks/useToast';

projects/app/src/components/support/user/team/TeamManageModal/InviteModal.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo, useState } from 'react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { ModalCloseButton, ModalBody, Box, ModalFooter, Button } from '@chakra-ui/react';
 import TagTextarea from '@/components/common/Textarea/TagTextarea';
 import MySelect from '@/components/Select';
@@ -57,8 +57,8 @@ const InviteModal = ({
         undefined,
         t('user.team.Invite Member Success Tip', {
           success: res.invite.length,
-          inValid: res.inValid.join(', '),
+          inValid: res.inValid.map((item) => item.username).join(', '),
-          inTeam: res.inTeam.join(', ')
+          inTeam: res.inTeam.map((item) => item.username).join(', ')
         })
       )();
     },

projects/app/src/components/support/user/team/TeamManageModal/index.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo, useState } from 'react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useQuery } from '@tanstack/react-query';
 import {
   getTeamList,
@@ -100,7 +100,9 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => {
     mutationFn: delRemoveMember,
     onSuccess() {
       refetchMembers();
-    }
+    },
+    successToast: t('user.team.Remove Member Success'),
+    errorToast: t('user.team.Remove Member Failed')
   });
   const { mutate: onLeaveTeam, isLoading: isLoadingLeaveTeam } = useRequest({
     mutationFn: async (teamId?: string) => {
@@ -306,7 +308,7 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => {
                         <Td display={'flex'} alignItems={'center'}>
                           <Avatar src={item.avatar} w={['18px', '22px']} />
                           <Box flex={'1 0 0'} w={0} ml={1} className={'textEllipsis'}>
-                            {item.memberUsername}
+                            {item.memberName}
                           </Box>
                         </Td>
                         <Td>{t(TeamMemberRoleMap[item.role]?.label || '')}</Td>
@@ -383,7 +385,7 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => {
                                           }),
                                         undefined,
                                         t('user.team.Remove Member Confirm Tip', {
-                                          username: item.memberUsername
+                                          username: item.memberName
                                         })
                                       )()
                                   }

projects/app/src/components/support/user/team/TeamMenu/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Box, Button, Flex, Image, useDisclosure, useTheme } from '@chakra-ui/react';
 import { useUserStore } from '@/web/support/user/useUserStore';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import MyTooltip from '@/components/MyTooltip';
 import dynamic from 'next/dynamic';
 import { feConfigs } from '@/web/common/system/staticData';

projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx

@@ -1,5 +1,5 @@
 import React, { useEffect } from 'react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import MyModal from '@/components/MyModal';
 import {
   Button,

projects/app/src/constants/dataset.ts

@@ -1,4 +1,7 @@
-import type { DatasetItemType } from '@/types/core/dataset';
+import type {
+  DatasetCollectionItemType,
+  DatasetItemType
+} from '@fastgpt/global/core/dataset/type.d';
 
 export const defaultDatasetDetail: DatasetItemType = {
   _id: '',
@@ -10,7 +13,7 @@ export const defaultDatasetDetail: DatasetItemType = {
   type: 'dataset',
   avatar: '/icon/logo.svg',
   name: '',
-  tags: '',
+  tags: [],
   permission: 'private',
   isOwner: false,
   canWrite: false,
@@ -22,3 +25,32 @@ export const defaultDatasetDetail: DatasetItemType = {
     maxToken: 3000
   }
 };
+
+export const defaultCollectionDetail: DatasetCollectionItemType = {
+  _id: '',
+  userId: '',
+  teamId: '',
+  tmbId: '',
+  datasetId: {
+    _id: '',
+    parentId: '',
+    userId: '',
+    teamId: '',
+    tmbId: '',
+    updateTime: new Date(),
+    type: 'dataset',
+    avatar: '/icon/logo.svg',
+    name: '',
+    tags: [],
+    permission: 'private',
+    vectorModel: 'text-embedding-ada-002'
+  },
+  parentId: '',
+  name: '',
+  type: 'file',
+  updateTime: new Date(),
+  metadata: {},
+  canWrite: false,
+  sourceName: '',
+  sourceId: ''
+};

projects/app/src/global/common/string/tools.ts

@@ -1,108 +0,0 @@
-import { getErrText } from '@fastgpt/global/common/error/utils';
-import { countPromptTokens } from '@/global/common/tiktoken';
-
-/* 
-    replace {{variable}} to value
-*/
-export function replaceVariable(text: string, obj: Record<string, string | number>) {
-  for (const key in obj) {
-    const val = obj[key];
-    if (!['string', 'number'].includes(typeof val)) continue;
-
-    text = text.replace(new RegExp(`{{(${key})}}`, 'g'), String(val));
-  }
-  return text || '';
-}
-
-/**
- * text split into chunks
- * maxLen - one chunk len. max: 3500
- * overlapLen - The size of the before and after Text
- * maxLen > overlapLen
- */
-export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen: number }) => {
-  const overlapLen = Math.floor(maxLen * 0.15); // Overlap length
-  const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
-
-  const stepReg: Record<number, RegExp> = {
-    0: /(\n\n)/g,
-    1: /([\n])/g,
-    2: /[。]|(?!<[^a-zA-Z])\.\s/g,
-    3: /([！？]|!\s|\?\s)/g,
-    4: /([；]|;\s)/g,
-    5: /([，]|,\s)/g
-  };
-
-  const splitTextRecursively = ({ text = '', step }: { text: string; step: number }) => {
-    if (text.length <= maxLen) {
-      return [text];
-    }
-    const reg = stepReg[step];
-
-    if (!reg) {
-      // use slice-maxLen to split text
-      const chunks: string[] = [];
-      let chunk = '';
-      for (let i = 0; i < text.length; i += maxLen - overlapLen) {
-        chunk = text.slice(i, i + maxLen);
-        chunks.push(chunk);
-      }
-      return chunks;
-    }
-
-    // split text by delimiters
-    const splitTexts = text
-      .replace(reg, `$1${tempMarker}`)
-      .split(`${tempMarker}`)
-      .filter((part) => part);
-
-    let chunks: string[] = [];
-    let preChunk = '';
-    let chunk = '';
-    for (let i = 0; i < splitTexts.length; i++) {
-      let text = splitTexts[i];
-      // chunk over size
-      if (text.length > maxLen) {
-        const innerChunks = splitTextRecursively({ text, step: step + 1 });
-        if (innerChunks.length === 0) continue;
-        // If the last chunk is too small, it is merged into the next chunk
-        if (innerChunks[innerChunks.length - 1].length <= maxLen * 0.5) {
-          text = innerChunks.pop() || '';
-          chunks = chunks.concat(innerChunks);
-        } else {
-          chunks = chunks.concat(innerChunks);
-          continue;
-        }
-      }
-
-      chunk += text;
-      // size over lapLen, push it to next chunk
-      if (chunk.length > maxLen - overlapLen) {
-        preChunk += text;
-      }
-      if (chunk.length >= maxLen) {
-        chunks.push(chunk);
-        chunk = preChunk;
-        preChunk = '';
-      }
-    }
-
-    if (chunk && !chunks[chunks.length - 1].endsWith(chunk)) {
-      chunks.push(chunk);
-    }
-    return chunks;
-  };
-
-  try {
-    const chunks = splitTextRecursively({ text, step: 0 });
-
-    const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
-
-    return {
-      chunks,
-      tokens
-    };
-  } catch (err) {
-    throw new Error(getErrText(err));
-  }
-};

projects/app/src/global/core/api/datasetReq.d.ts

@@ -2,7 +2,7 @@ import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core
 import type { RequestPaging } from '@/types';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { SearchTestItemType } from '@/types/core/dataset';
-import { DatasetChunkItemType, UploadChunkItemType } from '@fastgpt/global/core/dataset/type';
+import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type';
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
 
@@ -10,19 +10,11 @@ import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'
 export type DatasetUpdateParams = {
   id: string;
   parentId?: string;
-  tags?: string;
+  tags?: string[];
   name?: string;
   avatar?: string;
   permission?: `${PermissionTypeEnum}`;
 };
-export type CreateDatasetParams = {
-  parentId?: string;
-  name: string;
-  tags: string;
-  avatar: string;
-  vectorModel?: string;
-  type: `${DatasetTypeEnum}`;
-};
 
 export type SearchTestProps = {
   datasetId: string;
@@ -54,20 +46,6 @@ export type UpdateDatasetCollectionParams = {
 };
 
 /* ==== data ===== */
-export type SetOneDatasetDataProps = {
-  id?: string;
-  collectionId: string;
-  q?: string; // embedding content
-  a?: string; // bonus content
-};
-export type PushDataProps = {
-  collectionId: string;
-  data: DatasetChunkItemType[];
-  mode: `${TrainingModeEnum}`;
-  prompt?: string;
-  billId?: string;
-};
-
 export type GetDatasetDataListProps = RequestPaging & {
   searchText?: string;
   collectionId: string;

projects/app/src/global/core/dataset/api.d.ts

@@ -0,0 +1,35 @@
+import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
+
+/* ================= dataset ===================== */
+export type CreateDatasetParams = {
+  parentId?: string;
+  name: string;
+  tags: string;
+  avatar: string;
+  vectorModel?: string;
+  type: `${DatasetTypeEnum}`;
+};
+
+/* ================= collection ===================== */
+
+/* ================= data ===================== */
+export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & {
+  collectionId: string;
+};
+export type PushDatasetDataProps = {
+  collectionId: string;
+  data: PushDatasetDataChunkProps[];
+  mode: `${TrainingModeEnum}`;
+  prompt?: string;
+  billId?: string;
+};
+export type UpdateDatasetDataProps = {
+  id: string;
+  q?: string; // embedding content
+  a?: string; // bonus content
+  indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
+    dataId?: string; // pg data id
+  })[];
+};

projects/app/src/global/core/dataset/type.d.ts

@@ -1,5 +1,8 @@
 import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
-import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import {
+  DatasetCollectionSchemaType,
+  DatasetDataSchemaType
+} from '@fastgpt/global/core/dataset/type.d';
 
 /* ================= dataset ===================== */
 
@@ -11,7 +14,7 @@ export type DatasetCollectionsListItemType = {
   name: string;
   type: DatasetCollectionSchemaType['type'];
   updateTime: Date;
-  dataAmount?: number;
+  dataAmount: number;
   trainingAmount: number;
   metadata: DatasetCollectionSchemaType['metadata'];
   canWrite: boolean;
@@ -19,7 +22,10 @@ export type DatasetCollectionsListItemType = {
 
 /* ================= data ===================== */
 export type DatasetDataListItemType = {
-  id: string;
+  _id: string;
+  datasetId: string;
+  collectionId: string;
   q: string; // embedding content
   a: string; // bonus content
+  indexes: DatasetDataSchemaType['indexes'];
 };

projects/app/src/pages/_app.tsx

@@ -10,10 +10,10 @@ import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
 import { clientInitData, feConfigs } from '@/web/common/system/staticData';
 import { appWithTranslation, useTranslation } from 'next-i18next';
-import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
 import { useRouter } from 'next/router';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
+import { change2DefaultLng, setLngStore } from '@/web/common/utils/i18n';
 
 import 'nprogress/nprogress.css';
 import '@/web/styles/reset.scss';
@@ -57,6 +57,7 @@ function App({ Component, pageProps }: AppProps) {
         );
       setScripts(scripts || []);
     })();
+
     // add window error track
     window.onerror = function (msg, url) {
       window.umami?.track('windowError', {
@@ -76,23 +77,22 @@ function App({ Component, pageProps }: AppProps) {
   }, []);
 
   useEffect(() => {
-    hiId && localStorage.setItem('inviterId', hiId);
+    // get default language
-  }, [hiId]);
+    const targetLng = change2DefaultLng(i18n.language);
+    if (targetLng) {
+      setLngStore(targetLng);
+      router.replace(router.asPath, undefined, { locale: targetLng });
+    }
+  }, []);
 
   useEffect(() => {
-    const lang = getLangStore() || 'zh';
+    hiId && localStorage.setItem('inviterId', hiId);
-    i18n?.changeLanguage?.(lang);
+  }, [hiId]);
-    setLangStore(lang);
-
-    return () => {
-      setLastRoute(router.asPath);
-    };
-  }, [router.asPath]);
 
   return (
     <>
       <Head>
-        <title>{feConfigs?.systemTitle || 'AI'}</title>
+        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || 'GPT'}</title>
         <meta
           name="description"
           content="FastGPT is a knowledge-based question answering system built on the LLM. It offers out-of-the-box data processing and model invocation capabilities. Moreover, it allows for workflow orchestration through Flow visualization, thereby enabling complex question and answer scenarios!"

projects/app/src/pages/_document.tsx

@@ -2,7 +2,8 @@ import { Html, Head, Main, NextScript } from 'next/document';
 
 export default function Document() {
   return (
-    <Html lang="en">
+    <Html>
+      <title>{process.env.SYSTEM_NAME || 'FastGPT'}</title>
       <Head />
       <body>
         <Main />

projects/app/src/pages/_error.tsx

@@ -25,8 +25,9 @@ function Error() {
 
   return (
     <p>
-      部分系统不兼容，导致页面崩溃。如果可以，请联系作者，反馈下具体操作和页面。大部分是 苹果 的
+      部分系统不兼容，导致页面崩溃。如果可以，请联系作者，反馈下具体操作和页面。 大部分是 苹果 的
-      safari 浏览器导致，可以尝试更换 chrome 浏览器。
+      safari 浏览器导致，可以尝试更换 chrome
+      浏览器。或者是因为开了中文翻译导致，请检查并关闭中文翻译。
     </p>
   );
 }

projects/app/src/pages/account/components/ApiKeyTable.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import ApiKeyTable from '@/components/support/apikey/Table';
 
 const ApiKey = () => {

projects/app/src/pages/account/components/BillDetail.tsx

@@ -16,7 +16,7 @@ import dayjs from 'dayjs';
 import { BillSourceMap } from '@fastgpt/global/support/wallet/bill/constants';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void }) => {
   const { t } = useTranslation();
@@ -30,7 +30,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
       <ModalBody>
         <Flex alignItems={'center'} pb={4}>
           <Box flex={'0 0 80px'}>用户:</Box>
-          <Box>{bill.username}</Box>
+          <Box>{bill.memberName}</Box>
         </Flex>
         <Flex alignItems={'center'} pb={4}>
           <Box flex={'0 0 80px'}>订单号:</Box>

projects/app/src/pages/account/components/BillTable.tsx

@@ -55,7 +55,7 @@ const BillTable = () => {
         <Table>
           <Thead>
             <Tr>
-              <Th>{t('wallet.bill.bill username')}</Th>
+              <Th>{t('user.team.Member Name')}</Th>
               <Th>{t('user.Time')}</Th>
               <Th>{t('user.Source')}</Th>
               <Th>{t('user.Application Name')}</Th>
@@ -66,7 +66,7 @@ const BillTable = () => {
           <Tbody fontSize={'sm'}>
             {bills.map((item) => (
               <Tr key={item.id}>
-                <Td>{item.username}</Td>
+                <Td>{item.memberName}</Td>
                 <Td>{dayjs(item.time).format('YYYY/MM/DD HH:mm:ss')}</Td>
                 <Td>{BillSourceMap[item.source]}</Td>
                 <Td>{t(item.appName) || '-'}</Td>

projects/app/src/pages/account/components/Info.tsx

@@ -1,5 +1,14 @@
-import React, { useCallback, useRef, useState } from 'react';
+import React, { useCallback, useRef } from 'react';
-import { Box, Flex, Button, useDisclosure, useTheme, Divider, Select } from '@chakra-ui/react';
+import {
+  Box,
+  Flex,
+  Button,
+  useDisclosure,
+  useTheme,
+  Divider,
+  Select,
+  Input
+} from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
 import { UserUpdateParams } from '@/types/user';
 import { useToast } from '@/web/common/hooks/useToast';
@@ -16,10 +25,11 @@ import Loading from '@/components/Loading';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@/components/Icon';
 import MyTooltip from '@/components/MyTooltip';
-import { getLangStore, LangEnum, langMap, setLangStore } from '@/web/common/utils/i18n';
+import { langMap, setLngStore } from '@/web/common/utils/i18n';
 import { useRouter } from 'next/router';
 import MySelect from '@/components/Select';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
+import { putUpdateMemberName } from '@/web/support/user/team/api';
 
 const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu'));
 const PayModal = dynamic(() => import('./PayModal'), {
@@ -63,8 +73,6 @@ const UserInfo = () => {
     multiple: false
   });
 
-  const [language, setLanguage] = useState<`${LangEnum}`>(getLangStore());
-
   const onclickSave = useCallback(
     async (data: UserType) => {
       await updateUserInfo({
@@ -153,6 +161,27 @@ const UserInfo = () => {
         ml={[0, 10]}
         mt={[6, 0]}
       >
+        {feConfigs.isPlus && (
+          <Flex mb={4} alignItems={'center'} w={['85%', '300px']}>
+            <Box flex={'0 0 80px'}>{t('user.Member Name')}:&nbsp;</Box>
+            <Input
+              flex={1}
+              defaultValue={userInfo?.team?.memberName || 'Member'}
+              title={t('user.Edit name')}
+              borderColor={'transparent'}
+              pl={'10px'}
+              transform={'translateX(-11px)'}
+              maxLength={20}
+              onBlur={(e) => {
+                const val = e.target.value;
+                if (val === userInfo?.team?.memberName) return;
+                try {
+                  putUpdateMemberName(val);
+                } catch (error) {}
+              }}
+            />
+          </Flex>
+        )}
         <Flex alignItems={'center'} w={['85%', '300px']}>
           <Box flex={'0 0 80px'}>{t('user.Account')}:&nbsp;</Box>
           <Box flex={1}>{userInfo?.username}</Box>
@@ -167,17 +196,15 @@ const UserInfo = () => {
           <Box flex={'0 0 80px'}>{t('user.Language')}:&nbsp;</Box>
           <Box flex={'1 0 0'}>
             <MySelect
-              value={language}
+              value={i18n.language}
               list={Object.entries(langMap).map(([key, lang]) => ({
                 label: lang.label,
                 value: key
               }))}
               onchange={(val: any) => {
                 const lang = val;
-                setLangStore(lang);
+                setLngStore(lang);
-                setLanguage(lang);
+                router.replace(router.basePath, router.asPath, { locale: lang });
-                i18n?.changeLanguage?.(lang);
-                router.reload();
               }}
             />
           </Box>

projects/app/src/pages/account/components/OpenAIAccountModal.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { ModalBody, Box, Flex, Input, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useForm } from 'react-hook-form';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import type { UserType } from '@fastgpt/global/support/user/type.d';

projects/app/src/pages/account/components/PayModal.tsx

@@ -5,7 +5,7 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import Markdown from '@/components/Markdown';
 import MyModal from '@/components/MyModal';
 import { priceMd } from '@/web/common/system/staticData';

projects/app/src/pages/account/components/UpdatePswModal.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { ModalBody, Box, Flex, Input, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '@/components/MyModal';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useForm } from 'react-hook-form';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { updatePasswordByOld } from '@/web/support/user/api';

projects/app/src/pages/account/index.tsx

@@ -12,7 +12,7 @@ import Tabs from '@/components/Tabs';
 import UserInfo from './components/Info';
 import { serviceSideProps } from '@/web/common/utils/i18n';
 import { feConfigs } from '@/web/common/system/staticData';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import Script from 'next/script';
 
 const Promotion = dynamic(() => import('./components/Promotion'));
@@ -35,67 +35,64 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
   const { t } = useTranslation();
   const { userInfo, setUserInfo } = useUserStore();
 
-  const tabList = useMemo(
+  const tabList = [
-    () => [
+    {
-      {
+      icon: 'meLight',
-        icon: 'meLight',
+      label: t('user.Personal Information'),
-        label: t('user.Personal Information'),
+      id: TabEnum.info
-        id: TabEnum.info
+    },
-      },
+    ...(feConfigs?.isPlus
-      ...(feConfigs?.isPlus
+      ? [
-        ? [
+          {
-            {
+            icon: 'billRecordLight',
-              icon: 'billRecordLight',
+            label: t('user.Usage Record'),
-              label: t('user.Usage Record'),
+            id: TabEnum.bill
-              id: TabEnum.bill
+          }
-            }
+        ]
-          ]
+      : []),
-        : []),
+    ...(feConfigs?.show_promotion
-      ...(feConfigs?.show_promotion
+      ? [
-        ? [
+          {
-            {
+            icon: 'promotionLight',
-              icon: 'promotionLight',
+            label: t('user.Promotion Record'),
-              label: t('user.Promotion Record'),
+            id: TabEnum.promotion
-              id: TabEnum.promotion
+          }
-            }
+        ]
-          ]
+      : []),
-        : []),
+    ...(feConfigs?.show_pay && userInfo?.team.canWrite
-      ...(feConfigs?.show_pay && userInfo?.team.canWrite
+      ? [
-        ? [
+          {
-            {
+            icon: 'payRecordLight',
-              icon: 'payRecordLight',
+            label: t('user.Recharge Record'),
-              label: t('user.Recharge Record'),
+            id: TabEnum.pay
-              id: TabEnum.pay
+          }
-            }
+        ]
-          ]
+      : []),
-        : []),
+    ...(userInfo?.team.canWrite
-      ...(userInfo?.team.canWrite
+      ? [
-        ? [
+          {
-            {
+            icon: 'apikey',
-              icon: 'apikey',
+            label: t('user.apikey.key'),
-              label: t('user.apikey.key'),
+            id: TabEnum.apikey
-              id: TabEnum.apikey
+          }
-            }
+        ]
-          ]
+      : []),
-        : []),
+    ...(feConfigs.isPlus
-      ...(feConfigs.isPlus
+      ? [
-        ? [
+          {
-            {
+            icon: 'informLight',
-              icon: 'informLight',
+            label: t('user.Notice'),
-              label: t('user.Notice'),
+            id: TabEnum.inform
-              id: TabEnum.inform
+          }
-            }
+        ]
-          ]
+      : []),
-        : []),
 
-      {
+    {
-        icon: 'loginoutLight',
+      icon: 'loginoutLight',
-        label: t('user.Sign Out'),
+      label: t('user.Sign Out'),
-        id: TabEnum.loginout
+      id: TabEnum.loginout
-      }
+    }
-    ],
+  ];
-    [t, userInfo?.team.canWrite]
-  );
 
   const { openConfirm, ConfirmModal } = useConfirm({
     content: '确认退出登录？'

projects/app/src/pages/api/admin/initv46-2.ts

@@ -0,0 +1,169 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { delay } from '@/utils/tools';
+import { PgClient } from '@fastgpt/service/common/pg';
+import {
+  DatasetDataIndexTypeEnum,
+  PgDatasetTableName
+} from '@fastgpt/global/core/dataset/constant';
+
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
+
+let success = 0;
+/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { limit = 50 } = req.body as { limit: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+    success = 0;
+
+    try {
+      await Promise.allSettled([
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`),
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null
+        PgClient.query(
+          `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);`
+        ), // team_id varchar
+        PgClient.query(
+          `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);`
+        ), // tmb_id varchar
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null
+        PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null
+      ]);
+    } catch (error) {}
+
+    await initPgData();
+
+    jsonRes(res, {
+      data: await init(limit),
+      message:
+        '初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。'
+    });
+  } catch (error) {
+    console.log(error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+type PgItemType = {
+  id: string;
+  q: string;
+  a: string;
+  dataset_id: string;
+  collection_id: string;
+  team_id: string;
+  tmb_id: string;
+};
+
+async function initPgData() {
+  const limit = 10;
+  const { rows } = await PgClient.query<{ user_id: string }>(`
+  SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null';
+`);
+  console.log('init pg', rows.length);
+  let success = 0;
+  for (let i = 0; i < limit; i++) {
+    init(i);
+  }
+  async function init(index: number): Promise<any> {
+    const userId = rows[index]?.user_id;
+    if (!userId) return;
+    try {
+      const tmb = await getUserDefaultTeam({ userId });
+      // update pg
+      await PgClient.query(
+        `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id='null';`
+      );
+      console.log(++success);
+      init(index + limit);
+    } catch (error) {
+      if (error === 'default team not exist') {
+        return;
+      }
+      console.log(error);
+      await delay(1000);
+      return init(index);
+    }
+  }
+}
+
+async function init(limit: number): Promise<any> {
+  const { rows: idList } = await PgClient.query<{ id: string }>(
+    `SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL`
+  );
+
+  console.log('totalCount', idList.length);
+  if (idList.length === 0) return;
+
+  for (let i = 0; i < limit; i++) {
+    initData(i);
+  }
+
+  async function initData(index: number): Promise<any> {
+    const dataId = idList[index]?.id;
+    if (!dataId) {
+      console.log('done');
+      return;
+    }
+    // get limit data where data_id is null
+    const { rows } = await PgClient.query<PgItemType>(
+      `SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};`
+    );
+    const data = rows[0];
+    if (!data) {
+      console.log('done');
+      return;
+    }
+
+    let id = '';
+    try {
+      // create mongo data and update data_id
+      const { _id } = await MongoDatasetData.create({
+        teamId: data.team_id.trim(),
+        tmbId: data.tmb_id.trim(),
+        datasetId: data.dataset_id,
+        collectionId: data.collection_id,
+        q: data.q,
+        a: data.a,
+        indexes: [
+          {
+            defaultIndex: !data.a,
+            type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
+            dataId: data.id,
+            text: data.q
+          }
+        ]
+      });
+      id = _id;
+      // update pg data_id
+      await PgClient.query(
+        `UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};`
+      );
+
+      console.log(++success);
+      return initData(index + limit);
+    } catch (error) {
+      console.log(error);
+      console.log(data);
+
+      try {
+        if (id) {
+          await MongoDatasetData.findByIdAndDelete(id);
+        }
+      } catch (error) {}
+      await delay(500);
+      return initData(index);
+    }
+  }
+}

projects/app/src/pages/api/admin/initv46.ts

@@ -4,21 +4,14 @@ import { connectToDatabase } from '@/service/mongo';
 import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
 import {
   createDefaultTeam,
-  getTeamInfoByTmbId
+  getUserDefaultTeam
 } from '@fastgpt/service/support/user/team/controller';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
 import { UserModelSchema } from '@fastgpt/global/support/user/type';
 import { delay } from '@/utils/tools';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
-import {
-  DatasetCollectionSchemaType,
-  DatasetSchemaType,
-  DatasetTrainingSchemaType
-} from '@fastgpt/global/core/dataset/type';
 import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { connectionMongo } from '@fastgpt/service/common/mongo';
-import { Types } from 'mongoose';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { PgClient } from '@fastgpt/service/common/pg';
 import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
@@ -30,6 +23,7 @@ import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
 import { POST } from '@fastgpt/service/common/api/plusRequest';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
@@ -182,7 +176,7 @@ async function initMongoTeamId(limit: number) {
 
     async function init(userId: string): Promise<any> {
       try {
-        const tmb = await getTeamInfoByTmbId({ userId });
+        const tmb = await getUserDefaultTeam({ userId });
 
         await schema.updateMany(
           {
@@ -225,7 +219,7 @@ async function initDatasetAndApp() {
 }
 async function initCollectionFileTeam(limit: number) {
   /* init user default Team */
-  const DatasetFile = connectionMongo.connection.db.collection(`dataset.files`);
+  const DatasetFile = getGFSCollection('dataset');
   const matchWhere = {
     $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
   };
@@ -264,7 +258,7 @@ async function initCollectionFileTeam(limit: number) {
 
   async function init(userId: string): Promise<any> {
     try {
-      const tmb = await getTeamInfoByTmbId({
+      const tmb = await getUserDefaultTeam({
         userId
       });
 
@@ -295,8 +289,8 @@ async function initPgData() {
   // add column
   try {
     await Promise.all([
-      PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id CHAR(50);`),
+      PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`),
-      PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id CHAR(50);`),
+      PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`),
       PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`)
     ]);
   } catch (error) {
@@ -316,7 +310,7 @@ async function initPgData() {
     const userId = rows[index]?.user_id;
     if (!userId) return;
     try {
-      const tmb = await getTeamInfoByTmbId({ userId });
+      const tmb = await getUserDefaultTeam({ userId });
       // update pg
       await PgClient.query(
         `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;`

projects/app/src/pages/api/common/file/upload.ts

@@ -80,8 +80,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
   try {
     await connectToDatabase();
     const { userId, teamId, tmbId } = await authCert({ req, authToken: true });
-    console.log(req.body);
-
     const { files, bucketName, metadata } = await upload.doUpload(req, res);
 
     const upLoadResults = await Promise.all(

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -4,15 +4,16 @@ import { connectToDatabase } from '@/service/mongo';
 import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
 import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push';
 import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { messages } = req.body as CreateQuestionGuideParams;
+    const { messages, shareId } = req.body as CreateQuestionGuideParams;
-    const { tmbId, teamId } = await authCert({
+    const { tmbId, teamId } = await authCertAndShareId({
       req,
-      authToken: true
+      authToken: true,
+      shareId
     });
 
     const qgModel = global.qgModels[0];

projects/app/src/pages/api/core/dataset/allDataset.ts

@@ -3,10 +3,11 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import { getVectorModel } from '@/service/core/ai/model';
-import type { DatasetItemType } from '@/types/core/dataset';
+import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
 import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
 
+/* get all dataset by teamId or tmbId */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
@@ -20,7 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const data = datasets.map((item) => ({
       ...item.toJSON(),
-      tags: item.tags.join(' '),
       vectorModel: getVectorModel(item.vectorModel),
       canWrite: String(item.tmbId) === tmbId,
       isOwner: teamOwner || String(item.tmbId) === tmbId

projects/app/src/pages/api/core/dataset/collection/delById.ts

@@ -30,15 +30,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const collections = await findCollectionAndChild(collectionId, '_id metadata');
     const delIdList = collections.map((item) => item._id);
 
-    // delete pg data
-    await delDataByCollectionId({ collectionIds: delIdList });
-
     // delete training data
     await MongoDatasetTraining.deleteMany({
-      datasetCollectionId: { $in: delIdList },
+      collectionId: { $in: delIdList },
       teamId
     });
 
+    // delete pg data
+    await delDataByCollectionId({ collectionIds: delIdList });
+
     // delete file
     await Promise.all(
       collections.map((collection) => {

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -27,8 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     jsonRes<DatasetCollectionItemType>(res, {
       data: {
         ...collection,
-        datasetId: collection.datasetId._id,
+        canWrite,
-        canWrite
+        sourceName: collection?.name,
+        sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
       }
     });
   } catch (err) {

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -3,15 +3,15 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
 import { Types } from '@fastgpt/service/common/mongo';
-import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
+import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
 import { PagingData } from '@/types';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { countCollectionData } from '@/service/core/dataset/data/utils';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { startQueue } from '@/service/utils/tools';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
-import { getTeamInfoByTmbId } from '@fastgpt/service/support/user/team/controller';
+import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
+import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -30,7 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     // auth dataset and get my role
     const { tmbId } = await authDataset({ req, authToken: true, datasetId, per: 'r' });
-    const { canWrite } = await getTeamInfoByTmbId({ tmbId });
+    const { canWrite } = await authUserRole({ req, authToken: true });
 
     const match = {
       datasetId: new Types.ObjectId(datasetId),
@@ -59,7 +59,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
               ...item,
               dataAmount: 0,
               trainingAmount: 0,
-              canWrite // admin or owner can write
+              canWrite // admin or team owner can write
             }))
           ),
           total: await MongoDatasetCollection.countDocuments(match)
@@ -67,51 +67,75 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       });
     }
 
-    const collections: DatasetCollectionsListItemType[] = await MongoDatasetCollection.aggregate([
+    const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([
-      {
+      MongoDatasetCollection.aggregate([
-        $match: match
+        {
-      },
+          $match: match
-      {
+        },
-        $lookup: {
+        {
-          from: DatasetTrainingCollectionName,
+          $lookup: {
-          localField: '_id',
+            from: DatasetTrainingCollectionName,
-          foreignField: 'datasetCollectionId',
+            let: { id: '$_id' },
-          as: 'trainings_amount'
+            pipeline: [
+              {
+                $match: {
+                  $expr: {
+                    $eq: ['$collectionId', '$$id']
+                  }
+                }
+              },
+              { $project: { _id: 1 } }
+            ],
+            as: 'trainings'
+          }
+        },
+        {
+          $lookup: {
+            from: DatasetDataCollectionName,
+            let: { id: '$_id' },
+            pipeline: [
+              {
+                $match: {
+                  $expr: {
+                    $eq: ['$collectionId', '$$id']
+                  }
+                }
+              },
+              { $project: { _id: 1 } }
+            ],
+            as: 'datas'
+          }
+        },
+        // 统计子集合的数量和子训练的数量
+        {
+          $project: {
+            _id: 1,
+            parentId: 1,
+            tmbId: 1,
+            name: 1,
+            type: 1,
+            updateTime: 1,
+            trainingAmount: { $size: '$trainings' },
+            dataAmount: { $size: '$datas' },
+            metadata: 1
+          }
+        },
+        {
+          $sort: { updateTime: -1 }
+        },
+        {
+          $skip: (pageNum - 1) * pageSize
+        },
+        {
+          $limit: pageSize
         }
-      },
+      ]),
-      // 统计子集合的数量和子训练的数量
+      MongoDatasetCollection.countDocuments(match)
-      {
-        $project: {
-          _id: 1,
-          parentId: 1,
-          tmbId: 1,
-          name: 1,
-          type: 1,
-          updateTime: 1,
-          trainingAmount: { $size: '$trainings_amount' },
-          metadata: 1
-        }
-      },
-      {
-        $sort: { updateTime: -1 }
-      },
-      {
-        $skip: (pageNum - 1) * pageSize
-      },
-      {
-        $limit: pageSize
-      }
     ]);
 
-    const counts = await countCollectionData({
-      collectionIds: collections.map((item) => item._id),
-      datasetId
-    });
-
     const data = await Promise.all(
       collections.map(async (item, i) => ({
         ...item,
-        dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i],
         canWrite: String(item.tmbId) === tmbId || canWrite
       }))
     );
@@ -126,7 +150,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         pageNum,
         pageSize,
         data,
-        total: await MongoDatasetCollection.countDocuments(match)
+        total
       }
     });
   } catch (err) {

projects/app/src/pages/api/core/dataset/collection/paths.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import type { DatasetPathItemType } from '@/types/core/dataset';
+import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
 import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 
@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       parentId
     });
 
-    jsonRes<DatasetPathItemType[]>(res, {
+    jsonRes<ParentTreePathItemType[]>(res, {
       data: paths
     });
   } catch (err) {

projects/app/src/pages/api/core/dataset/create.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
-import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
+import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
 import { createDefaultCollection } from './collection/create';
 import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
 

projects/app/src/pages/api/core/dataset/data/delete.ts

@@ -1,10 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { PgClient } from '@fastgpt/service/common/pg';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
-import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
 import { connectToDatabase } from '@/service/mongo';
 import { authDatasetData } from '@/service/support/permission/auth/dataset';
+import { deleteDataByDataId } from '@/service/core/dataset/data/controller';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -20,11 +19,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     // 凭证校验
     await authDatasetData({ req, authToken: true, dataId, per: 'w' });
 
-    await PgClient.delete(PgDatasetTableName, {
+    await deleteDataByDataId(dataId);
-      where: [['id', dataId]]
-    });
 
-    jsonRes(res);
+    jsonRes(res, {
+      data: 'success'
+    });
   } catch (err) {
     console.log(err);
     jsonRes(res, {

projects/app/src/pages/api/core/dataset/data/detail.ts

@@ -13,12 +13,9 @@ export type Response = {
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    let { dataId } = req.query as {
+    const { dataId } = req.query as {
       dataId: string;
     };
-    if (!dataId) {
-      throw new Error('缺少参数');
-    }
 
     // 凭证校验
     const { datasetData } = await authDatasetData({ req, authToken: true, dataId, per: 'r' });