From 1a294c1fd316718bf49869914437342d95a0dd8b Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Mon, 9 Dec 2024 17:18:07 +0800 Subject: [PATCH] perf: load plugin groups code;perf: system plugin schema;fix: special variables replace;perf: retry cron app job (#3347) * perf: load plugin groups code * perf: system plugin schema * feat: retry cron app job * fix: special variables replace --- .../zh-cn/docs/development/upgrading/4815.md | 3 +- packages/global/common/system/utils.ts | 12 ++ packages/global/core/dataset/type.d.ts | 3 + .../global/core/workflow/runtime/utils.ts | 2 +- .../core/app/plugin/systemPluginSchema.ts | 3 +- packages/service/core/app/plugin/type.d.ts | 2 +- packages/service/core/app/schema.ts | 2 +- packages/service/core/dataset/schema.ts | 13 +++ packages/web/core/workflow/constants.ts | 8 +- projects/app/src/components/Layout/navbar.tsx | 2 +- .../account/components/AccountContainer.tsx | 1 + projects/app/src/pages/account/info/index.tsx | 1 - .../api/core/app/plugin/getPluginGroups.ts | 29 ----- .../app/src/service/common/system/cron.ts | 3 + projects/app/src/service/core/app/utils.ts | 105 ++++++++++-------- projects/app/src/web/core/app/api/plugin.ts | 9 +- 16 files changed, 106 insertions(+), 92 deletions(-) delete mode 100644 projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts diff --git a/docSite/content/zh-cn/docs/development/upgrading/4815.md b/docSite/content/zh-cn/docs/development/upgrading/4815.md index 741f5b35a..5e96d414d 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4815.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4815.md @@ -45,4 +45,5 @@ weight: 809 13. 修复 - 对话页面切换自动执行应用时，会误触发非自动执行应用。 14. 修复 - 语言播放鉴权问题。 15. 修复 - 插件应用知识库引用上限始终为 3000 -16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。 \ No newline at end of file +16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。 +17. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换） \ No newline at end of file diff --git a/packages/global/common/system/utils.ts b/packages/global/common/system/utils.ts index a6ca74d1b..8f79cb2f5 100644 --- a/packages/global/common/system/utils.ts +++ b/packages/global/common/system/utils.ts @@ -4,3 +4,15 @@ export const delay = (ms: number) => resolve(''); }, ms); }); + +export const retryFn = async (fn: () => Promise, retryTimes = 3): Promise => { + try { + return fn(); + } catch (error) { + if (retryTimes > 0) { + await delay(500); + return retryFn(fn, retryTimes - 1); + } + return Promise.reject(error); + } +}; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 960981a2b..d70cc5c0e 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -34,6 +34,9 @@ export type DatasetSchemaType = { inheritPermission: boolean; apiServer?: APIFileServer; + syncSchedule?: { cronString: string; timezone: string }; + syncNextTime?: Date; + // abandon externalReadUrl?: string; defaultPermission?: number; diff --git a/packages/global/core/workflow/runtime/utils.ts b/packages/global/core/workflow/runtime/utils.ts index 655cd6db3..d9744dfa2 100644 --- a/packages/global/core/workflow/runtime/utils.ts +++ b/packages/global/core/workflow/runtime/utils.ts @@ -321,7 +321,7 @@ export function replaceEditorVariable({ })(); const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g'); - text = text.replace(regex, formatVal); + text = text.replace(regex, () => formatVal); }); return text || ''; diff --git a/packages/service/core/app/plugin/systemPluginSchema.ts b/packages/service/core/app/plugin/systemPluginSchema.ts index 72ba7e658..2bfb2136a 100644 --- a/packages/service/core/app/plugin/systemPluginSchema.ts +++ b/packages/service/core/app/plugin/systemPluginSchema.ts @@ -10,8 +10,7 @@ const SystemPluginSchema = new Schema({ required: true }, isActive: { - type: Boolean, - required: true + type: Boolean }, inputConfig: { type: Array, diff --git a/packages/service/core/app/plugin/type.d.ts b/packages/service/core/app/plugin/type.d.ts index 90889bb9e..1275ec6d0 100644 --- a/packages/service/core/app/plugin/type.d.ts +++ b/packages/service/core/app/plugin/type.d.ts @@ -13,7 +13,7 @@ export type SystemPluginConfigSchemaType = { hasTokenFee: boolean; isActive: boolean; pluginOrder: number; - inputConfig: SystemPluginTemplateItemType['inputConfig']; + inputConfig?: SystemPluginTemplateItemType['inputConfig']; customConfig?: { name: string; diff --git a/packages/service/core/app/schema.ts b/packages/service/core/app/schema.ts index 71bb2ad33..d5c2791d7 100644 --- a/packages/service/core/app/schema.ts +++ b/packages/service/core/app/schema.ts @@ -121,6 +121,6 @@ const AppSchema = new Schema({ AppSchema.index({ teamId: 1, updateTime: -1 }); AppSchema.index({ teamId: 1, type: 1 }); -AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 }); +AppSchema.index({ scheduledTriggerConfig: 1, scheduledTriggerNextTime: -1 }); export const MongoApp = getMongoModel(AppCollectionName, AppSchema); diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 6312211ad..490703cd8 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -91,6 +91,18 @@ const DatasetSchema = new Schema({ type: Object }, + syncSchedule: { + cronString: { + type: String + }, + timezone: { + type: String + } + }, + syncNextTime: { + type: Date + }, + // abandoned externalReadUrl: { type: String @@ -100,6 +112,7 @@ const DatasetSchema = new Schema({ try { DatasetSchema.index({ teamId: 1 }); + DatasetSchema.index({ syncSchedule: 1, syncNextTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/web/core/workflow/constants.ts b/packages/web/core/workflow/constants.ts index d3a8b55b3..b41681d48 100644 --- a/packages/web/core/workflow/constants.ts +++ b/packages/web/core/workflow/constants.ts @@ -1,5 +1,6 @@ import { FlowNodeTemplateTypeEnum } from '@fastgpt/global/core/workflow/constants'; import { i18nT } from '../../i18n/utils'; +import type { PluginGroupSchemaType, TGroupType } from '../../../service/core/app/plugin/type'; export const workflowNodeTemplateList = [ { @@ -49,10 +50,7 @@ export const workflowNodeTemplateList = [ } ]; -export const systemPluginTemplateList: { - typeId: string; - typeName: string; -}[] = [ +export const systemPluginTemplateList: TGroupType[] = [ { typeId: FlowNodeTemplateTypeEnum.tools, typeName: i18nT('common:navbar.Tools') @@ -74,7 +72,7 @@ export const systemPluginTemplateList: { typeName: i18nT('common:common.Other') } ]; -export const defaultGroup = { +export const defaultGroup: PluginGroupSchemaType = { groupId: 'systemPlugin', groupAvatar: 'common/navbar/pluginLight', groupName: i18nT('common:core.module.template.System Plugin'), diff --git a/projects/app/src/components/Layout/navbar.tsx b/projects/app/src/components/Layout/navbar.tsx index 1cbbbaf09..f03df9905 100644 --- a/projects/app/src/components/Layout/navbar.tsx +++ b/projects/app/src/components/Layout/navbar.tsx @@ -115,7 +115,7 @@ const Navbar = ({ unread }: { unread: number }) => { borderRadius={'50%'} overflow={'hidden'} cursor={'pointer'} - onClick={() => router.push('/account')} + onClick={() => router.push('/account/info')} > diff --git a/projects/app/src/pages/account/components/AccountContainer.tsx b/projects/app/src/pages/account/components/AccountContainer.tsx index 0d1170dfa..68a4054a0 100644 --- a/projects/app/src/pages/account/components/AccountContainer.tsx +++ b/projects/app/src/pages/account/components/AccountContainer.tsx @@ -161,6 +161,7 @@ const AccountContainer = ({ m={'auto'} + w={'100%'} size={isPc ? 'md' : 'sm'} list={tabList.map((item) => ({ value: item.value, diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index 595b878a8..13e6da52b 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -118,7 +118,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { const standardPlan = teamPlanStatus?.standardConstants; const { isPc } = useSystem(); const { toast } = useToast(); - const router = useRouter(); const { isOpen: isOpenConversionModal, diff --git a/projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts b/projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts deleted file mode 100644 index 1a0fa5337..000000000 --- a/projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema'; -import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type'; -import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; - -export type getPluginGroupsQuery = {}; - -export type getPluginGroupsBody = {}; - -export type getPluginGroupsResponse = PluginGroupSchemaType[]; - -async function handler( - req: ApiRequestProps, - res: ApiResponseType -): Promise { - const pluginGroups = await MongoPluginGroups.find().sort({ groupOrder: 1 }); - - const result = pluginGroups.map((item) => ({ - groupId: item.groupId, - groupName: item.groupName, - groupAvatar: item.groupAvatar, - groupTypes: item.groupTypes, - groupOrder: item.groupOrder - })); - - return result; -} - -export default NextAPI(handler); diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts index 682691572..c6546f267 100644 --- a/projects/app/src/service/common/system/cron.ts +++ b/projects/app/src/service/common/system/cron.ts @@ -12,12 +12,14 @@ import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants' import { addHours } from 'date-fns'; import { getScheduleTriggerApp } from '@/service/core/app/utils'; +// Try to run train every minute const setTrainingQueueCron = () => { setCron('*/1 * * * *', () => { startTrainingQueue(); }); }; +// Clear tmp upload files every ten minutes const setClearTmpUploadFilesCron = () => { // Clear tmp upload files every ten minutes setCron('*/10 * * * *', () => { @@ -61,6 +63,7 @@ const clearInvalidDataCron = () => { }); }; +// Run app timer trigger every hour const scheduleTriggerAppCron = () => { setCron('0 */1 * * *', async () => { if ( diff --git a/projects/app/src/service/core/app/utils.ts b/projects/app/src/service/core/app/utils.ts index b68c4b310..1584ae808 100644 --- a/projects/app/src/service/core/app/utils.ts +++ b/projects/app/src/service/core/app/utils.ts @@ -3,7 +3,7 @@ import { pushChatUsage } from '@/service/support/wallet/usage/push'; import { defaultApp } from '@/web/core/app/constants'; import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time'; import { getNanoid } from '@fastgpt/global/common/string/tools'; -import { delay } from '@fastgpt/global/common/system/utils'; +import { delay, retryFn } from '@fastgpt/global/common/system/utils'; import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants'; import { getWorkflowEntryNodeIds, @@ -18,63 +18,72 @@ import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; export const getScheduleTriggerApp = async () => { // 1. Find all the app - const apps = await MongoApp.find({ - scheduledTriggerConfig: { $ne: null }, - scheduledTriggerNextTime: { $lte: new Date() } + const apps = await retryFn(() => { + return MongoApp.find({ + scheduledTriggerConfig: { $ne: null }, + scheduledTriggerNextTime: { $lte: new Date() } + }); }); // 2. Run apps await Promise.allSettled( apps.map(async (app) => { - if (!app.scheduledTriggerConfig) return; - // random delay 0 ~ 60s - await delay(Math.floor(Math.random() * 60 * 1000)); - const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId); - try { - const { flowUsages } = await dispatchWorkFlow({ - chatId: getNanoid(), - user, - mode: 'chat', - runningAppInfo: { - id: String(app._id), - teamId: String(app.teamId), - tmbId: String(app.tmbId) - }, - uid: String(app.tmbId), - runtimeNodes: storeNodes2RuntimeNodes(app.modules, getWorkflowEntryNodeIds(app.modules)), - runtimeEdges: initWorkflowEdgeStatus(app.edges), - variables: {}, - query: [ - { - type: ChatItemValueTypeEnum.text, - text: { - content: app.scheduledTriggerConfig?.defaultPrompt + if (!app.scheduledTriggerConfig) return; + // random delay 0 ~ 60s + await delay(Math.floor(Math.random() * 60 * 1000)); + const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId); + + await retryFn(async () => { + if (!app.scheduledTriggerConfig) return; + + const { flowUsages } = await dispatchWorkFlow({ + chatId: getNanoid(), + user, + mode: 'chat', + runningAppInfo: { + id: String(app._id), + teamId: String(app.teamId), + tmbId: String(app.tmbId) + }, + uid: String(app.tmbId), + runtimeNodes: storeNodes2RuntimeNodes( + app.modules, + getWorkflowEntryNodeIds(app.modules) + ), + runtimeEdges: initWorkflowEdgeStatus(app.edges), + variables: {}, + query: [ + { + type: ChatItemValueTypeEnum.text, + text: { + content: app.scheduledTriggerConfig?.defaultPrompt + } } - } - ], - chatConfig: defaultApp.chatConfig, - histories: [], - stream: false, - maxRunTimes: WORKFLOW_MAX_RUN_TIMES - }); - pushChatUsage({ - appName: app.name, - appId: app._id, - teamId: String(app.teamId), - tmbId: String(app.tmbId), - source: UsageSourceEnum.cronJob, - flowUsages + ], + chatConfig: defaultApp.chatConfig, + histories: [], + stream: false, + maxRunTimes: WORKFLOW_MAX_RUN_TIMES + }); + pushChatUsage({ + appName: app.name, + appId: app._id, + teamId: String(app.teamId), + tmbId: String(app.tmbId), + source: UsageSourceEnum.cronJob, + flowUsages + }); }); + + // update next time + app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone( + app.scheduledTriggerConfig + ); + await app.save(); } catch (error) { - addLog.error('Schedule trigger error', error); + addLog.warn('Schedule trigger error', { error }); } - - // update next time - app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone(app.scheduledTriggerConfig); - await app.save(); - - return; }) ); }; diff --git a/projects/app/src/web/core/app/api/plugin.ts b/projects/app/src/web/core/app/api/plugin.ts index 399eef108..ed13b625e 100644 --- a/projects/app/src/web/core/app/api/plugin.ts +++ b/projects/app/src/web/core/app/api/plugin.ts @@ -14,6 +14,8 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; import { GetSystemPluginTemplatesBody } from '@/pages/api/core/app/plugin/getSystemPluginTemplates'; import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { defaultGroup } from '@fastgpt/web/core/workflow/constants'; /* ============ team plugin ============== */ export const getTeamPlugTemplates = (data?: ListAppBody) => @@ -41,8 +43,11 @@ export const getTeamPlugTemplates = (data?: ListAppBody) => export const getSystemPlugTemplates = (data: GetSystemPluginTemplatesBody) => POST('/core/app/plugin/getSystemPluginTemplates', data); -export const getPluginGroups = () => - GET('/core/app/plugin/getPluginGroups'); +export const getPluginGroups = () => { + return useSystemStore.getState()?.feConfigs?.isPlus + ? GET('/proApi/core/app/plugin/getPluginGroups') + : Promise.resolve([defaultGroup]); +}; export const getSystemPluginPaths = (parentId: ParentIdType) => { if (!parentId) return Promise.resolve([]);