Update docs and response tag in share page (#694)

* perf: chunk index show

* share response

* perf: vector query

* web printFinger

* remove log

* fix: bucket name

* perf: training schema

* perf: sort index

docSite/content/docs/commercial/intro.md

@@ -33,7 +33,7 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独
 
 ## 商业版软件价格
 
-FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)
+FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)
 
 **共有服务**
 
@@ -46,8 +46,7 @@ FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。
 {{< table "table-hover table-striped-columns" >}}
 | 部署方式 | 特有服务 | 上线时长 | 价格 |
 | ---- | ---- | ---- | ---- |
-| Sealos公有云部署 |  1. 6个版本的升级服务。<br>2. 赠送 8000 元 Sealos 云资源额度。 | 半天 | 10000元/年 | 
+| Sealos全托管 |  1. 有效期内免费升级。<br>2. 免运维服务&数据库。 |  半天  | 3000元起/月（3个月起）<br>或<br>30000元起/年  |
-| Sealos全托管 |  1. 有效期内免费升级。<br>2. 免运维服务&数据库。<br>3. 赠送 10000 元 Sealos 云资源额度。 |  半天  | 3000元起/月（3个月起）<br>或<br>30000元起/年  |
 | 自有服务器-单机版 |  1. 6个版本的升级服务。 | 14天内 |  60000元/套（不限时长） | 
 | 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）|
 {{< /table >}}

docSite/content/docs/development/openapi/auth.md

@@ -25,7 +25,7 @@ FastGPT 的 API Key **有 2 类**，一类是全局通用的 key (无法直接
 
 |        通用key               |           应用特定 key            |
 | --------------------- | --------------------- |
-| ![](/imgs/fastgpt-api2.png) | ![](/imgs/fastgpt-api.png) |
+| ![](/imgs/fastgpt-api2.jpg) | ![](/imgs/fastgpt-api.jpg) |
 
 ## 基本配置
 

packages/global/common/file/constants.ts

@@ -1,5 +1,10 @@
 export enum BucketNameEnum {
   dataset = 'dataset'
 }
+export const bucketNameMap = {
+  [BucketNameEnum.dataset]: {
+    label: 'common.file.bucket.dataset'
+  }
+};
 
 export const FileBaseUrl = '/api/common/file/read';

packages/global/common/file/read/index.ts

@@ -20,13 +20,13 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer })
 
     const viewport = page.getViewport({ scale: 1 });
     const pageHeight = viewport.height;
-    const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内
+    const headerThreshold = pageHeight * 0.95;
-    const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内
+    const footerThreshold = pageHeight * 0.05;
 
     const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
       return (
         !token.transform ||
-        (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
+        (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
       );
     });
 

packages/service/common/file/upload/multer.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { customAlphabet } from 'nanoid';
 import multer from 'multer';
 import path from 'path';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import fs from 'fs';
 
 const nanoid = customAlphabet('1234567890abcdef', 12);
@@ -45,6 +45,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
             return reject(error);
           }
 
+          // check bucket name
+          const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+          if (bucketName && !bucketNameMap[bucketName]) {
+            return reject('BucketName is invalid');
+          }
+
           resolve({
             ...req.body,
             files:

packages/service/common/vectorStore/controller.ts

@@ -48,7 +48,7 @@ export const updateDatasetDataVector = async ({
   // get vector
   const { vectors, tokens } = await getVectorsByText({
     model,
-    input: [query]
+    input: query
   });
 
   await getVectorObj().update({

packages/service/core/ai/embedding/index.ts

@@ -2,7 +2,7 @@ import { getAIApi } from '../config';
 
 export type GetVectorProps = {
   model: string;
-  input: string | string[];
+  input: string;
 };
 
 // text to vector
@@ -10,24 +10,13 @@ export async function getVectorsByText({
   model = 'text-embedding-ada-002',
   input
 }: GetVectorProps) {
-  if (typeof input === 'string' && !input) {
+  if (!input) {
     return Promise.reject({
       code: 500,
       message: 'input is empty'
     });
-  } else if (Array.isArray(input)) {
-    for (let i = 0; i < input.length; i++) {
-      if (!input[i]) {
-        return Promise.reject({
-          code: 500,
-          message: 'input array is empty'
-        });
-      }
-    }
-  }
-  if (typeof input === 'string') {
-    input = [input];
   }
+
   try {
     // 获取 chatAPI
     const ai = getAIApi();
@@ -36,7 +25,7 @@ export async function getVectorsByText({
     const result = await ai.embeddings
       .create({
         model,
-        input
+        input: [input]
       })
       .then(async (res) => {
         if (!res.data) {
@@ -47,6 +36,7 @@ export async function getVectorsByText({
           // @ts-ignore
           return Promise.reject(res.data?.err?.message || 'Embedding API Error');
         }
+
         return {
           tokens: res.usage.total_tokens || 0,
           vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))

packages/service/core/dataset/training/schema.ts

@@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({
 });
 
 try {
+  TrainingDataSchema.index({ weight: -1 });
   TrainingDataSchema.index({ lockTime: 1 });
   TrainingDataSchema.index({ datasetId: 1 });
   TrainingDataSchema.index({ collectionId: 1 });

packages/web/common/system/utils.ts

@@ -0,0 +1,9 @@
+import FingerprintJS from '@fingerprintjs/fingerprintjs';
+
+const fpPromise = FingerprintJS.load();
+
+export const getUserFingerprint = async () => {
+  const fp = await fpPromise;
+  const result = await fp.get();
+  console.log(result.visitorId);
+};

packages/web/package.json

@@ -2,9 +2,6 @@
   "name": "@fastgpt/web",
   "version": "1.0.0",
   "dependencies": {
-    "@fastgpt/global": "workspace:*",
-    "joplin-turndown-plugin-gfm": "^1.0.12",
-    "turndown": "^7.1.2",
     "@chakra-ui/anatomy": "^2.2.1",
     "@chakra-ui/icons": "^2.1.1",
     "@chakra-ui/next-js": "^2.1.5",
@@ -13,16 +10,20 @@
     "@chakra-ui/system": "^2.6.1",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
+    "@fastgpt/global": "workspace:*",
+    "@fingerprintjs/fingerprintjs": "^4.2.1",
     "@monaco-editor/react": "^4.6.0",
+    "i18next": "^22.5.1",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "next-i18next": "^13.3.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "i18next": "^22.5.1",
+    "react-i18next": "^12.3.1",
-    "next-i18next": "^13.3.0",
+    "turndown": "^7.1.2"
-    "react-i18next": "^12.3.1"
   },
   "devDependencies": {
-    "@types/turndown": "^5.0.4",
     "@types/react": "18.2.0",
-    "@types/react-dom": "18.2.0"
+    "@types/react-dom": "18.2.0",
+    "@types/turndown": "^5.0.4"
   }
 }

pnpm-lock.yaml

@@ -163,6 +163,9 @@ importers:
       '@fastgpt/global':
         specifier: workspace:*
         version: link:../global
+      '@fingerprintjs/fingerprintjs':
+        specifier: ^4.2.1
+        version: registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1
       '@monaco-editor/react':
         specifier: ^4.6.0
         version: registry.npmmirror.com/@monaco-editor/react@4.6.0(monaco-editor@0.45.0)(react-dom@18.2.0)(react@18.2.0)
@@ -3836,6 +3839,14 @@ packages:
       - supports-color
     dev: true
 
+  registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1:
+    resolution: {integrity: sha512-uW+GVUNTgCXbVPEbgnbf5Aor22e1dyYR0JRwdUiZBaikfxr7KlhV9y0aahA1FB99fEeQVvhCEvTcPIFSYTy9Pw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@fingerprintjs/fingerprintjs/-/fingerprintjs-4.2.1.tgz}
+    name: '@fingerprintjs/fingerprintjs'
+    version: 4.2.1
+    dependencies:
+      tslib: registry.npmmirror.com/tslib@2.6.2
+    dev: false
+
   registry.npmmirror.com/@humanwhocodes/config-array@0.11.13:
     resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz}
     name: '@humanwhocodes/config-array'

projects/app/public/locales/en/common.json

@@ -522,9 +522,13 @@
         },
         "score": {
           "embedding": "Embedding",
+          "embedding desc": "",
           "fullText": "Full text",
+          "fullText desc": "",
           "reRank": "ReRank",
-          "rrf": "RRF Merge"
+          "reRank desc": "",
+          "rrf": "RRF Merge",
+          "rrf desc": ""
         },
         "search mode": "Search Mode"
       },
@@ -874,8 +878,6 @@
     "QPM": "QPM",
     "QPM Tips": "The maximum number of queries per IP address per minute",
     "QPM is empty": "QPM is empty",
-    "Response Detail": "Quote",
-    "Response Detail tips": "Whether detailed data such as references to be returned",
     "token auth": "Token Auth",
     "token auth Tips": "Identity verification server address. If this value is set, the server will be specified to send a request for identity verification before each session",
     "token auth use cases": "Review the authentication instructions"
@@ -903,6 +905,12 @@
     "Update Your Plugin": "Update Plugin"
   },
   "support": {
+    "outlink": {
+      "share": {
+        "Response Quote": "Show Quote",
+        "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document."
+      }
+    },
     "user": {
       "Price": "Price",
       "auth": {

projects/app/public/locales/zh/common.json

@@ -878,8 +878,6 @@
     "QPM": "",
     "QPM Tips": "每个 IP 每分钟最多提问多少次",
     "QPM is empty": "QPM 不能为空",
-    "Response Detail": "返回详情",
-    "Response Detail tips": "是否需要返回详情（引用内容，调用时间等，不会返回预设提示词和完整上下文）",
     "token auth": "身份验证",
     "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会向指定服务器发送一个请求，进行身份校验",
     "token auth use cases": "查看身份验证使用说明"
@@ -907,6 +905,12 @@
     "Update Your Plugin": "更新插件"
   },
   "support": {
+    "outlink": {
+      "share": {
+        "Response Quote": "返回引用",
+        "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档"
+      }
+    },
     "user": {
       "Price": "计费标准",
       "auth": {

projects/app/src/components/ChatBox/ResponseTags.tsx

@@ -133,6 +133,7 @@ const ResponseTags = ({
           </Flex>
         </>
       )}
+      {!isShare && (
         <Flex alignItems={'center'} mt={3} flexWrap={'wrap'}>
           {quoteList.length > 0 && (
             <MyTooltip label="查看引用">
@@ -180,7 +181,8 @@ const ResponseTags = ({
               {t('core.chat.response.Read complete response')}
             </Tag>
           </MyTooltip>
-
+        </Flex>
+      )}
       {!!quoteModalData && (
         <QuoteModal
           {...quoteModalData}
@@ -192,13 +194,8 @@ const ResponseTags = ({
         <ContextModal context={contextModalData} onClose={() => setContextModalData(undefined)} />
       )}
       {isOpenWholeModal && (
-          <WholeResponseModal
+        <WholeResponseModal response={responseData} isShare={isShare} onClose={onCloseWholeModal} />
-            response={responseData}
-            isShare={isShare}
-            onClose={onCloseWholeModal}
-          />
       )}
-      </Flex>
     </>
   );
 };

projects/app/src/components/core/dataset/QuoteItem.tsx

@@ -126,6 +126,8 @@ const QuoteItem = ({
       >
         <Flex alignItems={'center'} mb={3}>
           {score?.primaryScore && (
+            <>
+              {canViewSource ? (
                 <MyTooltip label={t(SearchScoreTypeMap[score.primaryScore.type]?.desc)}>
                   <Flex
                     px={'12px'}
@@ -140,7 +142,12 @@ const QuoteItem = ({
                     fontSize={'sm'}
                   >
                     <Box>#{score.primaryScore.index + 1}</Box>
-                <Box borderRightColor={'primary.700'} borderRightWidth={'1px'} h={'14px'} mx={2} />
+                    <Box
+                      borderRightColor={'primary.700'}
+                      borderRightWidth={'1px'}
+                      h={'14px'}
+                      mx={2}
+                    />
                     <Box>
                       {t(SearchScoreTypeMap[score.primaryScore.type]?.label)}
                       {SearchScoreTypeMap[score.primaryScore.type]?.showScore
@@ -149,8 +156,26 @@ const QuoteItem = ({
                     </Box>
                   </Flex>
                 </MyTooltip>
+              ) : (
+                <Flex
+                  px={'12px'}
+                  py={'1px'}
+                  mr={4}
+                  borderRadius={'md'}
+                  color={'primary.700'}
+                  bg={'primary.50'}
+                  borderWidth={'1px'}
+                  borderColor={'primary.200'}
+                  alignItems={'center'}
+                  fontSize={'sm'}
+                >
+                  <Box>#{score.primaryScore.index + 1}</Box>
+                </Flex>
               )}
-          {score.secondaryScore.map((item, i) => (
+            </>
+          )}
+          {canViewSource &&
+            score.secondaryScore.map((item, i) => (
               <MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc)}>
                 <Box fontSize={'xs'} mr={3}>
                   <Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}>

projects/app/src/global/core/dataset/type.d.ts

@@ -28,5 +28,6 @@ export type DatasetDataListItemType = {
   collectionId: string;
   q: string; // embedding content
   a: string; // bonus content
+  chunkIndex?: number;
   indexes: DatasetDataSchemaType['indexes'];
 };

projects/app/src/pages/api/core/dataset/data/list.ts

@@ -32,7 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     };
 
     const [data, total] = await Promise.all([
-      MongoDatasetData.find(match, '_id datasetId collectionId q a indexes')
+      MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex indexes')
         .sort({ chunkIndex: 1, updateTime: -1 })
         .skip((pageNum - 1) * pageSize)
         .limit(pageSize)

projects/app/src/pages/api/v1/embeddings.ts

@@ -10,6 +10,7 @@ import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
 
 type Props = GetVectorProps & {
+  input: string | string[];
   billId?: string;
 };
 
@@ -22,6 +23,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       throw new Error('input is nor array or string');
     }
 
+    const query = Array.isArray(input) ? input[0] : input;
+
     const { teamId, tmbId, apikey, authType } = await authCert({
       req,
       authToken: true,
@@ -30,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 
     await authTeamBalance(teamId);
 
-    const { tokens, vectors } = await getVectorsByText({ input, model });
+    const { tokens, vectors } = await getVectorsByText({ input: query, model });
 
     jsonRes(res, {
       data: {

projects/app/src/pages/app/detail/components/OutLink/Share.tsx

@@ -96,7 +96,7 @@ const Share = ({ appId }: { appId: string }) => {
             <Tr>
               <Th>名称</Th>
               <Th>金额消耗</Th>
-              <Th>返回详情</Th>
+              <Th>返回引用</Th>
               {feConfigs?.isPlus && (
                 <>
                   <Th>IP限流（人/分钟）</Th>
@@ -373,8 +373,8 @@ function EditLinkModal({
 
         <Flex alignItems={'center'} mt={4}>
           <Flex flex={'0 0 90px'} alignItems={'center'}>
-            {t('outlink.Response Detail')}
+            {t('support.outlink.share.Response Quote')}
-            <MyTooltip label={t('outlink.Response Detail tips' || '')}>
+            <MyTooltip label={t('support.outlink.share.Response Quote tips' || '')}>
               <QuestionOutlineIcon ml={1} />
             </MyTooltip>
           </Flex>

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -307,7 +307,7 @@ const DataCard = () => {
           >
             <Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
               <Box border={theme.borders.base} px={2} fontSize={'sm'} mr={1} borderRadius={'md'}>
-                # {index + 1}
+                # {item.chunkIndex ?? '-'}
               </Box>
               <Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
                 ID:{item._id}

projects/app/src/service/core/dataset/data/controller.ts

@@ -295,7 +295,7 @@ export async function searchDatasetData(props: {
   const embeddingRecall = async ({ query, limit }: { query: string; limit: number }) => {
     const { vectors, tokens } = await getVectorsByText({
       model,
-      input: [query]
+      input: query
     });
 
     const { results } = await recallFromVectorStore({

projects/app/src/service/events/generateVector.ts

@@ -26,6 +26,8 @@ export async function generateVector(): Promise<any> {
   if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
   global.vectorQueueLen++;
 
+  const start = Date.now();
+
   // get training data
   const {
     data,
@@ -154,6 +156,8 @@ export async function generateVector(): Promise<any> {
     await MongoDatasetTraining.findByIdAndDelete(data._id);
     reduceQueue();
     generateVector();
+
+    console.log(`embedding finished, time: ${Date.now() - start}ms`);
   } catch (err: any) {
     reduceQueue(true);
     // log

projects/app/src/utils/service/core/chat/index.ts

@@ -1,19 +1,16 @@
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
+import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
 
-export function selectShareResponse({
+export const selectShareResponse = ({
   responseData = []
 }: {
   responseData?: ChatHistoryItemResType[];
-}) {
+}) => {
-  const filedList = [
+  const filedList = ['quoteList', 'moduleType'];
-    'moduleType',
+  const filterModuleTypeList: any[] = [FlowNodeTypeEnum.chatNode];
-    'moduleName',
+  return responseData
-    'moduleLogo',
+    .filter((item) => filterModuleTypeList.includes(item.moduleType))
-    'runningTime',
+    .map((item) => {
-    'quoteList',
-    'question'
-  ];
-  return responseData.map((item) => {
       const obj: Record<string, any> = {};
       for (let key in item) {
         if (filedList.includes(key)) {
@@ -23,4 +20,4 @@ export function selectShareResponse({
       }
       return obj as ChatHistoryItemResType;
     });
-}
+};