gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: worker un exit (#624)

Browse Source
This commit is contained in:
Archer 2023-12-18 20:35:17 +08:00 committed by GitHub
parent 703583fff7
commit b14a1db2f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 58 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/global/common/string/textSplitter.ts
View File

@ -232,7 +232,7 @@ export const splitText2Chunks = (props: {
step: 0, step: 0,
lastText: '', lastText: '',
mdTitle: '' mdTitle: ''
}).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block
const tokens = countTokens const tokens = countTokens
? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0) ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0)

3
packages/service/common/string/markdown.ts
View File

@ -13,9 +13,12 @@ export const htmlToMarkdown = (html?: string | null) =>
const worker = new Worker(getWorkerPath('html2md')); const worker = new Worker(getWorkerPath('html2md'));
worker.on('message', (md: string) => { worker.on('message', (md: string) => {
worker.terminate();
resolve(simpleMarkdownText(md)); resolve(simpleMarkdownText(md));
}); });
worker.on('error', (err) => { worker.on('error', (err) => {
worker.terminate();
reject(err); reject(err);
}); });

35
pnpm-lock.yaml generated
View File

@ -84,15 +84,9 @@ importers:
dayjs: dayjs:
specifier: ^1.11.7 specifier: ^1.11.7
version: registry.npmmirror.com/dayjs@1.11.7 version: registry.npmmirror.com/dayjs@1.11.7
domino-ext:
specifier: ^2.1.4
version: registry.npmmirror.com/domino-ext@2.1.4
encoding: encoding:
specifier: ^0.1.13 specifier: ^0.1.13
version: registry.npmmirror.com/encoding@0.1.13 version: registry.npmmirror.com/encoding@0.1.13
joplin-turndown-plugin-gfm:
specifier: ^1.0.12
version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12
jsonwebtoken: jsonwebtoken:
specifier: ^9.0.2 specifier: ^9.0.2
version: registry.npmmirror.com/jsonwebtoken@9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2
@ -117,9 +111,6 @@ importers:
tunnel: tunnel:
specifier: ^0.0.6 specifier: ^0.0.6
version: registry.npmmirror.com/tunnel@0.0.6 version: registry.npmmirror.com/tunnel@0.0.6
turndown:
specifier: ^7.1.2
version: registry.npmmirror.com/turndown@7.1.2
devDependencies: devDependencies:
'@types/cookie': '@types/cookie':
specifier: ^0.5.2 specifier: ^0.5.2
@ -6765,18 +6756,6 @@ packages:
domelementtype: registry.npmmirror.com/domelementtype@2.3.0 domelementtype: registry.npmmirror.com/domelementtype@2.3.0
dev: false dev: false
registry.npmmirror.com/domino-ext@2.1.4:
resolution: {integrity: sha512-t8piRI9Qahd4V/NqnCcqdBWsQ4OYeOvcTuoHl38Pzk9OJJ/UiCYHA2jX2fACmBtDlSMiWa0uR524KuLEAMc/3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino-ext/-/domino-ext-2.1.4.tgz}
name: domino-ext
version: 2.1.4
dev: false
registry.npmmirror.com/domino@2.1.6:
resolution: {integrity: sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino/-/domino-2.1.6.tgz}
name: domino
version: 2.1.6
dev: false
registry.npmmirror.com/dompurify@3.0.3: registry.npmmirror.com/dompurify@3.0.3:
resolution: {integrity: sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.3.tgz} resolution: {integrity: sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.3.tgz}
name: dompurify name: dompurify
@ -8793,12 +8772,6 @@ packages:
set-function-name: registry.npmmirror.com/set-function-name@2.0.1 set-function-name: registry.npmmirror.com/set-function-name@2.0.1
dev: true dev: true
registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12:
resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/joplin-turndown-plugin-gfm/-/joplin-turndown-plugin-gfm-1.0.12.tgz}
name: joplin-turndown-plugin-gfm
version: 1.0.12
dev: false
registry.npmmirror.com/js-sdsl@4.4.2: registry.npmmirror.com/js-sdsl@4.4.2:
resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz} resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz}
name: js-sdsl name: js-sdsl
@ -12193,14 +12166,6 @@ packages:
engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'}
dev: false dev: false
registry.npmmirror.com/turndown@7.1.2:
resolution: {integrity: sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/turndown/-/turndown-7.1.2.tgz}
name: turndown
version: 7.1.2
dependencies:
domino: registry.npmmirror.com/domino@2.1.6
dev: false
registry.npmmirror.com/type-check@0.4.0: registry.npmmirror.com/type-check@0.4.0:
resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/type-check/-/type-check-0.4.0.tgz} resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/type-check/-/type-check-0.4.0.tgz}
name: type-check name: type-check

2
projects/app/src/components/Markdown/chat/Guide.tsx
View File

@ -23,7 +23,7 @@ function MyLink(e: any) {
<Box as={'li'} mb={1}> <Box as={'li'} mb={1}>
<Box <Box
as={'span'} as={'span'}
color={'blue.500'} color={'blue.700'}
textDecoration={'underline'} textDecoration={'underline'}
cursor={'pointer'} cursor={'pointer'}
onClick={() => { onClick={() => {

2
projects/app/src/components/SideTabs/index.tsx
View File

@ -44,7 +44,7 @@ const SideTabs = ({ list, size = 'md', activeId, onChange, ...props }: Props) =>
alignItems={'center'} alignItems={'center'}
{...(activeId === item.id {...(activeId === item.id
? { ? {
bg: ' blue.200 !important', bg: ' blue.100 !important',
fontWeight: 'bold', fontWeight: 'bold',
color: 'blue.600 ', color: 'blue.600 ',
cursor: 'default' cursor: 'default'

8
projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx
View File

@ -1,11 +1,5 @@
import React from 'react'; import React from 'react';
import { import { SmoothStepEdge, EdgeLabelRenderer, EdgeProps, getSmoothStepPath } from 'reactflow';
SmoothStepEdge,
EdgeLabelRenderer,
EdgeProps,
getSmoothStepPath,
MarkerType
} from 'reactflow';
import { Flex } from '@chakra-ui/react'; import { Flex } from '@chakra-ui/react';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';

84
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
View File

@ -289,28 +289,30 @@ export const PreviewFileOrChunk = () => {
overflow={'overlay'} overflow={'overlay'}
px={[4, 8]} px={[4, 8]}
my={4} my={4}
contentEditable // contentEditable
dangerouslySetInnerHTML={{ __html: previewFile.rawText }} // dangerouslySetInnerHTML={{ __html: previewFile.rawText }}
fontSize={'sm'} fontSize={'sm'}
whiteSpace={'pre-wrap'} whiteSpace={'pre-wrap'}
wordBreak={'break-all'} wordBreak={'break-all'}
onBlur={(e) => { // onBlur={(e) => {
// @ts-ignore // // @ts-ignore
const val = e.target.innerText; // const val = e.target.innerText;
setReShowRePreview(true); // setReShowRePreview(true);
setFiles((state) => // setFiles((state) =>
state.map((file) => // state.map((file) =>
file.id === previewFile.id // file.id === previewFile.id
? { // ? {
...file, // ...file,
text: val // text: val
} // }
: file // : file
) // )
); // );
}} // }}
/> >
{previewFile.rawText}
</Box>
</Box> </Box>
) : ( ) : (
<Box pt={[3, 6]}> <Box pt={[3, 6]}>
@ -365,49 +367,9 @@ export const PreviewFileOrChunk = () => {
}} }}
/> />
</Flex> </Flex>
<Box <Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
px={4} {chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q}
fontSize={'sm'} </Box>
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
contentEditable={!chunk.a}
dangerouslySetInnerHTML={{
__html: chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q
}}
onBlur={(e) => {
// @ts-ignore
const val = e.target.innerText;
/* delete file */
if (val === '') {
setFiles((state) =>
state.map((stateFile) =>
stateFile.id === file.id
? {
...file,
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
}
: stateFile
)
);
} else {
// update chunk
setFiles((stateFiles) =>
stateFiles.map((stateFile) =>
file.id === stateFile.id
? {
...stateFile,
chunks: stateFile.chunks.map((chunk, index) => ({
...chunk,
index: i === index ? val : chunk.q
}))
}
: stateFile
)
);
}
}}
/>
</Box> </Box>
)) ))
)} )}

20
projects/app/src/web/styles/theme.ts
View File

@ -284,16 +284,16 @@ export const theme = extendTheme({
1000: '#121416' 1000: '#121416'
}, },
blue: { blue: {
50: '#f0f7ff', 50: '#F0F4FF',
100: '#EBF7FD', 100: '#E1EAFF',
200: '#d6e8ff', 200: '#C5D7FF',
300: '#adceff', 300: '#94B5FF',
400: '#85b1ff', 400: '#5E8FFF',
500: '#4e83fd', 500: '#487FFF',
600: '#3370ff', 600: '#3370FF',
700: '#2152d9', 700: '#2B5FD9',
800: '#1237b3', 800: '#2450B5',
900: '#07228c' 900: '#1D4091'
}, },
myRead: { myRead: {
600: '#ff4d4f' 600: '#ff4d4f'

32
worker/html2md.js
View File

@ -20,22 +20,26 @@ parentPort?.on('message', (html) => {
}); });
const html2md = (html) => { const html2md = (html) => {
const window = domino.createWindow(html); try {
const document = window.document; const window = domino.createWindow(html);
const document = window.document;
turndownService.remove(['i', 'script', 'iframe']); turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', { turndownService.addRule('codeBlock', {
filter: 'pre', filter: 'pre',
replacement(_, node) { replacement(_, node) {
const content = node.textContent?.trim() || ''; const content = node.textContent?.trim() || '';
// @ts-ignore // @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || ''; const codeName = node?._attrsByQName?.class?.data?.trim() || '';
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`; return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
} }
}); });
turndownService.use(turndownPluginGfm.gfm); turndownService.use(turndownPluginGfm.gfm);
return turndownService.turndown(document); return turndownService.turndown(document);
} catch (error) {
return '';
}
}; };