gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: chunk filter

Browse Source
This commit is contained in:
archer 2023-08-07 11:04:32 +08:00
parent 7fe20ef041
commit b6e156db26
No known key found for this signature in database
GPG Key ID: 569A5660D2379E28
3 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/src/pages/kb/detail/components/Import/Chunk.tsx
View File

@ -16,7 +16,7 @@ import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file'; import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query'; import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb'; import { postKbDataFromList } from '@/api/plugins/kb';
import { splitText_token } from '@/utils/file'; import { splitText2Chunks } from '@/utils/file';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { formatPrice } from '@/utils/user'; import { formatPrice } from '@/utils/user';
import { vectorModelList } from '@/store/static'; import { vectorModelList } from '@/store/static';
@ -96,7 +96,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
})(); })();
if (icon && text) { if (icon && text) {
const splitRes = splitText_token({ const splitRes = splitText2Chunks({
text: text, text: text,
maxLen: chunkLen maxLen: chunkLen
}); });
@ -178,7 +178,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
const onRePreview = useCallback(async () => { const onRePreview = useCallback(async () => {
try { try {
const splitRes = files.map((item) => const splitRes = files.map((item) =>
splitText_token({ splitText2Chunks({
text: item.text, text: item.text,
maxLen: chunkLen maxLen: chunkLen
}) })

6
client/src/pages/kb/detail/components/Import/QA.tsx
View File

@ -5,7 +5,7 @@ import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file'; import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query'; import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb'; import { postKbDataFromList } from '@/api/plugins/kb';
import { splitText_token } from '@/utils/file'; import { splitText2Chunks } from '@/utils/file';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { formatPrice } from '@/utils/user'; import { formatPrice } from '@/utils/user';
import { qaModelList } from '@/store/static'; import { qaModelList } from '@/store/static';
@ -86,7 +86,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
})(); })();
if (icon && text) { if (icon && text) {
const splitRes = splitText_token({ const splitRes = splitText2Chunks({
text: text, text: text,
maxLen: chunkLen maxLen: chunkLen
}); });
@ -169,7 +169,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
const onRePreview = useCallback(async () => { const onRePreview = useCallback(async () => {
try { try {
const splitRes = files.map((item) => const splitRes = files.map((item) =>
splitText_token({ splitText2Chunks({
text: item.text, text: item.text,
maxLen: chunkLen maxLen: chunkLen
}) })

2
client/src/utils/file.ts
View File

@ -147,7 +147,7 @@ export const fileDownload = ({
* overlapLen - The size of the before and after Text * overlapLen - The size of the before and after Text
* maxLen > overlapLen * maxLen > overlapLen
*/ */
export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => { export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
try { try {