From b6e156db26e9602f3f72d19f78ef5ba4284037c0 Mon Sep 17 00:00:00 2001
From: archer <545436317@qq.com>
Date: Mon, 7 Aug 2023 11:04:32 +0800
Subject: [PATCH] perf: chunk filter

---
 client/src/pages/kb/detail/components/Import/Chunk.tsx | 6 +++---
 client/src/pages/kb/detail/components/Import/QA.tsx    | 6 +++---
 client/src/utils/file.ts                               | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/client/src/pages/kb/detail/components/Import/Chunk.tsx b/client/src/pages/kb/detail/components/Import/Chunk.tsx
index 432deb181..5d63db7de 100644
--- a/client/src/pages/kb/detail/components/Import/Chunk.tsx
+++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx
@@ -16,7 +16,7 @@ import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
-import { splitText_token } from '@/utils/file';
+import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
 import { vectorModelList } from '@/store/static';
@@ -96,7 +96,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
             })();
 
             if (icon && text) {
-              const splitRes = splitText_token({
+              const splitRes = splitText2Chunks({
                 text: text,
                 maxLen: chunkLen
               });
@@ -178,7 +178,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
   const onRePreview = useCallback(async () => {
     try {
       const splitRes = files.map((item) =>
-        splitText_token({
+        splitText2Chunks({
           text: item.text,
           maxLen: chunkLen
         })
diff --git a/client/src/pages/kb/detail/components/Import/QA.tsx b/client/src/pages/kb/detail/components/Import/QA.tsx
index 294313b7a..71ebc5851 100644
--- a/client/src/pages/kb/detail/components/Import/QA.tsx
+++ b/client/src/pages/kb/detail/components/Import/QA.tsx
@@ -5,7 +5,7 @@ import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
-import { splitText_token } from '@/utils/file';
+import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
 import { qaModelList } from '@/store/static';
@@ -86,7 +86,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
             })();
 
             if (icon && text) {
-              const splitRes = splitText_token({
+              const splitRes = splitText2Chunks({
                 text: text,
                 maxLen: chunkLen
               });
@@ -169,7 +169,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
   const onRePreview = useCallback(async () => {
     try {
       const splitRes = files.map((item) =>
-        splitText_token({
+        splitText2Chunks({
           text: item.text,
           maxLen: chunkLen
         })
diff --git a/client/src/utils/file.ts b/client/src/utils/file.ts
index 085621a7f..c5f82a8c9 100644
--- a/client/src/utils/file.ts
+++ b/client/src/utils/file.ts
@@ -147,7 +147,7 @@ export const fileDownload = ({
  * overlapLen - The size of the before and after Text
  * maxLen > overlapLen
  */
-export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => {
+export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
   const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
 
   try {