mirror of
https://github.com/EthanMarti/infio-copilot.git
synced 2026-01-16 08:21:55 +00:00
add local embed
This commit is contained in:
parent
cd65d6b3de
commit
65c5df3d22
@ -61,6 +61,7 @@
|
|||||||
"@electric-sql/pglite": "0.2.14",
|
"@electric-sql/pglite": "0.2.14",
|
||||||
"@google/genai": "^1.2.0",
|
"@google/genai": "^1.2.0",
|
||||||
"@google/generative-ai": "^0.21.0",
|
"@google/generative-ai": "^0.21.0",
|
||||||
|
"@huggingface/transformers": "^3.6.1",
|
||||||
"@langchain/core": "^0.3.26",
|
"@langchain/core": "^0.3.26",
|
||||||
"@lexical/clipboard": "^0.17.1",
|
"@lexical/clipboard": "^0.17.1",
|
||||||
"@lexical/react": "^0.17.1",
|
"@lexical/react": "^0.17.1",
|
||||||
@ -74,6 +75,7 @@
|
|||||||
"@radix-ui/react-tooltip": "^1.1.3",
|
"@radix-ui/react-tooltip": "^1.1.3",
|
||||||
"@tanstack/react-query": "^5.56.2",
|
"@tanstack/react-query": "^5.56.2",
|
||||||
"@types/mermaid": "^9.2.0",
|
"@types/mermaid": "^9.2.0",
|
||||||
|
"@xenova/transformers": "^2.17.2",
|
||||||
"axios": "^1.8.3",
|
"axios": "^1.8.3",
|
||||||
"chokidar": "^4.0.3",
|
"chokidar": "^4.0.3",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
@ -119,6 +121,7 @@
|
|||||||
"remark-gfm": "^4.0.0",
|
"remark-gfm": "^4.0.0",
|
||||||
"shell-env": "^4.0.1",
|
"shell-env": "^4.0.1",
|
||||||
"simple-git": "^3.27.0",
|
"simple-git": "^3.27.0",
|
||||||
|
"smart-embed-model": "^1.0.7",
|
||||||
"string-similarity": "^4.0.4",
|
"string-similarity": "^4.0.4",
|
||||||
"styled-components": "^6.1.19",
|
"styled-components": "^6.1.19",
|
||||||
"uuid": "^10.0.0",
|
"uuid": "^10.0.0",
|
||||||
|
|||||||
947
pnpm-lock.yaml
generated
947
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,8 @@
|
|||||||
import { CheckSquare, Clock, CopyPlus, MessageSquare, Pencil, Search, Sparkles, Square, Trash2 } from 'lucide-react'
|
import { CheckSquare, Clock, CopyPlus, Globe, MessageSquare, Pencil, Search, Sparkles, Square, Trash2 } from 'lucide-react'
|
||||||
import { Notice } from 'obsidian'
|
import { Notice } from 'obsidian'
|
||||||
import React, { useMemo, useRef, useState } from 'react'
|
import React, { useMemo, useRef, useState } from 'react'
|
||||||
|
|
||||||
|
import { useSettings } from '../../contexts/SettingsContext'
|
||||||
import { useChatHistory } from '../../hooks/use-chat-history'
|
import { useChatHistory } from '../../hooks/use-chat-history'
|
||||||
import { t } from '../../lang/helpers'
|
import { t } from '../../lang/helpers'
|
||||||
import { ChatConversationMeta } from '../../types/chat'
|
import { ChatConversationMeta } from '../../types/chat'
|
||||||
@ -29,6 +30,15 @@ const ChatHistoryView = ({
|
|||||||
// search term
|
// search term
|
||||||
const [searchTerm, setSearchTerm] = useState('')
|
const [searchTerm, setSearchTerm] = useState('')
|
||||||
|
|
||||||
|
const { settings } = useSettings()
|
||||||
|
|
||||||
|
const currentWorkspace = React.useMemo(() => {
|
||||||
|
return settings.workspace || 'vault'
|
||||||
|
}, [settings.workspace])
|
||||||
|
|
||||||
|
// workspace filter state
|
||||||
|
const [filterByWorkspace, setFilterByWorkspace] = useState(false)
|
||||||
|
|
||||||
// editing conversation id
|
// editing conversation id
|
||||||
const [editingConversationId, setEditingConversationId] = useState<string | null>(null)
|
const [editingConversationId, setEditingConversationId] = useState<string | null>(null)
|
||||||
|
|
||||||
@ -62,16 +72,33 @@ const ChatHistoryView = ({
|
|||||||
setSearchTerm(e.target.value)
|
setSearchTerm(e.target.value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// toggle workspace filter
|
||||||
|
const toggleWorkspaceFilter = () => {
|
||||||
|
setFilterByWorkspace(!filterByWorkspace)
|
||||||
|
}
|
||||||
|
|
||||||
// filter conversations list
|
// filter conversations list
|
||||||
const filteredConversations = useMemo(() => {
|
const filteredConversations = useMemo(() => {
|
||||||
if (!searchTerm.trim()) {
|
console.log('filteredConversations', chatList)
|
||||||
return chatList
|
let filtered = chatList
|
||||||
}
|
|
||||||
return chatList.filter(
|
// Apply search filter
|
||||||
|
if (searchTerm.trim()) {
|
||||||
|
filtered = filtered.filter(
|
||||||
conversation =>
|
conversation =>
|
||||||
conversation.title.toLowerCase().includes(searchTerm.toLowerCase())
|
conversation.title.toLowerCase().includes(searchTerm.toLowerCase())
|
||||||
)
|
)
|
||||||
}, [chatList, searchTerm])
|
}
|
||||||
|
|
||||||
|
// Apply workspace filter
|
||||||
|
if (filterByWorkspace) {
|
||||||
|
filtered = filtered.filter(
|
||||||
|
conversation => conversation.workspace === currentWorkspace
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filtered
|
||||||
|
}, [chatList, searchTerm, filterByWorkspace, currentWorkspace])
|
||||||
|
|
||||||
// toggle selection mode
|
// toggle selection mode
|
||||||
const toggleSelectionMode = () => {
|
const toggleSelectionMode = () => {
|
||||||
@ -285,6 +312,18 @@ const ChatHistoryView = ({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* workspace filter */}
|
||||||
|
<div className="infio-chat-history-workspace-filter">
|
||||||
|
<button
|
||||||
|
onClick={toggleWorkspaceFilter}
|
||||||
|
className={`infio-chat-history-workspace-filter-btn ${filterByWorkspace ? 'active' : ''}`}
|
||||||
|
title={filterByWorkspace ? '显示所有对话' : '只显示当前工作区对话'}
|
||||||
|
>
|
||||||
|
<Globe size={14} />
|
||||||
|
当前工作区
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* conversations list */}
|
{/* conversations list */}
|
||||||
<div className="infio-chat-history-list">
|
<div className="infio-chat-history-list">
|
||||||
{filteredConversations.length === 0 ? (
|
{filteredConversations.length === 0 ? (
|
||||||
@ -353,6 +392,11 @@ const ChatHistoryView = ({
|
|||||||
{formatDate(conversation.updatedAt)}
|
{formatDate(conversation.updatedAt)}
|
||||||
</div>
|
</div>
|
||||||
<div className="infio-chat-history-conversation-title">{conversation.title}</div>
|
<div className="infio-chat-history-conversation-title">{conversation.title}</div>
|
||||||
|
{conversation.workspace && (
|
||||||
|
<div className="infio-chat-history-workspace">
|
||||||
|
工作区: {conversation.workspace}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
{!selectionMode && (
|
{!selectionMode && (
|
||||||
<div className="infio-chat-history-actions">
|
<div className="infio-chat-history-actions">
|
||||||
@ -426,6 +470,7 @@ const ChatHistoryView = ({
|
|||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-filter-btn,
|
||||||
.infio-chat-history-cleanup-btn,
|
.infio-chat-history-cleanup-btn,
|
||||||
.infio-chat-history-selection-btn {
|
.infio-chat-history-selection-btn {
|
||||||
display: flex !important;
|
display: flex !important;
|
||||||
@ -444,12 +489,14 @@ const ChatHistoryView = ({
|
|||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-filter-btn:hover,
|
||||||
.infio-chat-history-cleanup-btn:hover,
|
.infio-chat-history-cleanup-btn:hover,
|
||||||
.infio-chat-history-selection-btn:hover {
|
.infio-chat-history-selection-btn:hover {
|
||||||
background-color: var(--background-modifier-hover, #f5f5f5);
|
background-color: var(--background-modifier-hover, #f5f5f5);
|
||||||
border-color: var(--background-modifier-border-hover, #d0d0d0);
|
border-color: var(--background-modifier-border-hover, #d0d0d0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-filter-btn.active,
|
||||||
.infio-chat-history-selection-btn.active {
|
.infio-chat-history-selection-btn.active {
|
||||||
background-color: var(--interactive-accent, #007acc);
|
background-color: var(--interactive-accent, #007acc);
|
||||||
color: var(--text-on-accent, #ffffff);
|
color: var(--text-on-accent, #ffffff);
|
||||||
@ -539,7 +586,6 @@ const ChatHistoryView = ({
|
|||||||
border: 1px solid var(--background-modifier-border);
|
border: 1px solid var(--background-modifier-border);
|
||||||
border-radius: var(--radius-s);
|
border-radius: var(--radius-s);
|
||||||
padding: 6px 12px;
|
padding: 6px 12px;
|
||||||
margin-bottom: var(--size-4-3);
|
|
||||||
transition: all 0.2s ease;
|
transition: all 0.2s ease;
|
||||||
height: 36px;
|
height: 36px;
|
||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
@ -675,6 +721,13 @@ const ChatHistoryView = ({
|
|||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-workspace {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: 11px;
|
||||||
|
margin-top: 2px;
|
||||||
|
opacity: 0.8;
|
||||||
|
}
|
||||||
|
|
||||||
.infio-chat-history-actions {
|
.infio-chat-history-actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
gap: 4px;
|
gap: 4px;
|
||||||
@ -768,6 +821,38 @@ const ChatHistoryView = ({
|
|||||||
.infio-chat-history-cancel-btn:hover {
|
.infio-chat-history-cancel-btn:hover {
|
||||||
background-color: var(--background-modifier-hover);
|
background-color: var(--background-modifier-hover);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-workspace-filter {
|
||||||
|
display: flex;
|
||||||
|
justify-content: flex-start;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-workspace-filter-btn {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
background-color: transparent;
|
||||||
|
border: 1px solid var(--background-modifier-border);
|
||||||
|
color: var(--text-muted);
|
||||||
|
padding: 6px 12px;
|
||||||
|
border-radius: var(--radius-s);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: var(--font-ui-small);
|
||||||
|
transition: all 0.2s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-workspace-filter-btn:hover {
|
||||||
|
background-color: var(--background-modifier-hover);
|
||||||
|
color: var(--text-normal);
|
||||||
|
}
|
||||||
|
|
||||||
|
.infio-chat-history-workspace-filter-btn.active {
|
||||||
|
background-color: var(--interactive-accent);
|
||||||
|
color: var(--text-on-accent);
|
||||||
|
border-color: var(--interactive-accent);
|
||||||
|
}
|
||||||
`}
|
`}
|
||||||
</style>
|
</style>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@ -879,7 +879,7 @@ const Chat = forwardRef<ChatRef, ChatProps>((props, ref) => {
|
|||||||
// Execute the transformation using the TransEngine
|
// Execute the transformation using the TransEngine
|
||||||
const transformationResult = await transEngine.runTransformation({
|
const transformationResult = await transEngine.runTransformation({
|
||||||
filePath: toolArgs.path,
|
filePath: toolArgs.path,
|
||||||
transformationType: transformationType,
|
transformationType: transformationType as TransformationType,
|
||||||
model: {
|
model: {
|
||||||
provider: settings.applyModelProvider,
|
provider: settings.applyModelProvider,
|
||||||
modelId: settings.applyModelId,
|
modelId: settings.applyModelId,
|
||||||
|
|||||||
@ -148,7 +148,7 @@ const SearchView = () => {
|
|||||||
minSimilarity: 0.3,
|
minSimilarity: 0.3,
|
||||||
})
|
})
|
||||||
|
|
||||||
setInsightResults(results)
|
setInsightResults(results as any)
|
||||||
setSearchResults([])
|
setSearchResults([])
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@ -60,7 +60,8 @@ const WorkspaceEditModal = ({
|
|||||||
|
|
||||||
// 直接使用 Obsidian 的内置接口获取所有标签
|
// 直接使用 Obsidian 的内置接口获取所有标签
|
||||||
try {
|
try {
|
||||||
const tagsObject = app.metadataCache.getTags() // 获取所有标签 {'#tag1': 2, '#tag2': 4}
|
// @ts-ignore
|
||||||
|
const tagsObject = app.metadataCache.getTags() as Record<string, number> // 获取所有标签 {'#tag1': 2, '#tag2': 4}
|
||||||
const tags = Object.keys(tagsObject).sort()
|
const tags = Object.keys(tagsObject).sort()
|
||||||
setAvailableTags(tags)
|
setAvailableTags(tags)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@ -112,5 +112,5 @@ export function getRulesSection(
|
|||||||
if (mode === 'learn') {
|
if (mode === 'learn') {
|
||||||
return getLearnModeRulesSection(cwd, searchTool);
|
return getLearnModeRulesSection(cwd, searchTool);
|
||||||
}
|
}
|
||||||
return getObsidianRulesSection(mode, cwd, searchTool, supportsComputerUse, diffStrategy, experiments);
|
return getObsidianRulesSection(mode, cwd, searchTool);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -21,8 +21,5 @@ By waiting for and carefully considering the user's response after each tool use
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function getToolUseGuidelinesSection(mode?: string): string {
|
export function getToolUseGuidelinesSection(mode?: string): string {
|
||||||
if (mode === 'learn') {
|
|
||||||
return getLearnModeToolUseGuidelines();
|
|
||||||
}
|
|
||||||
return getDefaultToolUseGuidelines();
|
return getDefaultToolUseGuidelines();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -61,7 +61,7 @@ export type ToolName = keyof typeof TOOL_DISPLAY_NAMES
|
|||||||
|
|
||||||
// Tool helper functions
|
// Tool helper functions
|
||||||
export function getToolName(toolConfig: string | readonly [ToolName, ...unknown[]]): ToolName {
|
export function getToolName(toolConfig: string | readonly [ToolName, ...unknown[]]): ToolName {
|
||||||
return typeof toolConfig === "string" ? toolConfig : toolConfig[0]
|
return typeof toolConfig === "string" ? toolConfig as ToolName : toolConfig[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getToolOptions(toolConfig: string | readonly [ToolName, ...unknown[]]): unknown {
|
export function getToolOptions(toolConfig: string | readonly [ToolName, ...unknown[]]): unknown {
|
||||||
|
|||||||
@ -981,8 +981,9 @@ export class TransEngine {
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case TransformationType.DENSE_SUMMARY:
|
||||||
case TransformationType.CONCISE_DENSE_SUMMARY:
|
// 新的摘要类型不需要特殊的后处理,保持原样
|
||||||
|
break;
|
||||||
case TransformationType.HIERARCHICAL_SUMMARY:
|
case TransformationType.HIERARCHICAL_SUMMARY:
|
||||||
// 新的摘要类型不需要特殊的后处理,保持原样
|
// 新的摘要类型不需要特殊的后处理,保持原样
|
||||||
break;
|
break;
|
||||||
@ -1237,7 +1238,7 @@ export class TransEngine {
|
|||||||
const cacheResult = await this.checkDatabaseCache(
|
const cacheResult = await this.checkDatabaseCache(
|
||||||
fileMetadata.sourcePath,
|
fileMetadata.sourcePath,
|
||||||
fileMetadata.sourceMtime,
|
fileMetadata.sourceMtime,
|
||||||
TransformationType.CONCISE_DENSE_SUMMARY
|
TransformationType.DENSE_SUMMARY
|
||||||
)
|
)
|
||||||
|
|
||||||
if (cacheResult.foundCache && cacheResult.result.success && cacheResult.result.result) {
|
if (cacheResult.foundCache && cacheResult.result.success && cacheResult.result.result) {
|
||||||
@ -1273,7 +1274,7 @@ export class TransEngine {
|
|||||||
// 保存到数据库
|
// 保存到数据库
|
||||||
await this.saveResultToDatabase(
|
await this.saveResultToDatabase(
|
||||||
summary,
|
summary,
|
||||||
TransformationType.CONCISE_DENSE_SUMMARY,
|
TransformationType.DENSE_SUMMARY,
|
||||||
fileMetadata.sourcePath,
|
fileMetadata.sourcePath,
|
||||||
fileMetadata.sourceMtime,
|
fileMetadata.sourceMtime,
|
||||||
'document'
|
'document'
|
||||||
@ -1295,7 +1296,7 @@ export class TransEngine {
|
|||||||
const messages: RequestMessage[] = [
|
const messages: RequestMessage[] = [
|
||||||
{
|
{
|
||||||
role: 'system',
|
role: 'system',
|
||||||
content: CONCISE_DENSE_SUMMARY_PROMPT
|
content: DENSE_SUMMARY_PROMPT
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
role: 'user',
|
role: 'user',
|
||||||
@ -1308,7 +1309,7 @@ export class TransEngine {
|
|||||||
throw new Error(`生成摘要失败: ${result.error.message}`)
|
throw new Error(`生成摘要失败: ${result.error.message}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.postProcessResult(result.value, TransformationType.CONCISE_DENSE_SUMMARY)
|
return this.postProcessResult(result.value, TransformationType.DENSE_SUMMARY)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import { ChatConversationMeta } from '../../../types/chat'
|
|||||||
import { AbstractJsonRepository } from '../base'
|
import { AbstractJsonRepository } from '../base'
|
||||||
import { CHAT_DIR, ROOT_DIR } from '../constants'
|
import { CHAT_DIR, ROOT_DIR } from '../constants'
|
||||||
import { EmptyChatTitleException } from '../exception'
|
import { EmptyChatTitleException } from '../exception'
|
||||||
|
import { WorkspaceManager } from '../workspace/WorkspaceManager'
|
||||||
|
|
||||||
import {
|
import {
|
||||||
CHAT_SCHEMA_VERSION,
|
CHAT_SCHEMA_VERSION,
|
||||||
@ -16,27 +17,34 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
ChatConversation,
|
ChatConversation,
|
||||||
ChatConversationMeta
|
ChatConversationMeta
|
||||||
> {
|
> {
|
||||||
constructor(app: App) {
|
private workspaceManager?: WorkspaceManager
|
||||||
|
|
||||||
|
constructor(app: App, workspaceManager?: WorkspaceManager) {
|
||||||
super(app, `${ROOT_DIR}/${CHAT_DIR}`)
|
super(app, `${ROOT_DIR}/${CHAT_DIR}`)
|
||||||
|
this.workspaceManager = workspaceManager
|
||||||
}
|
}
|
||||||
|
|
||||||
protected generateFileName(chat: ChatConversation): string {
|
protected generateFileName(chat: ChatConversation): string {
|
||||||
// Format: v{schemaVersion}_{title}_{updatedAt}_{id}.json
|
// 新格式: v{schemaVersion}_{title}_{updatedAt}_{id}_{workspaceId}.json
|
||||||
|
// 如果没有工作区,使用 'vault' 作为默认值
|
||||||
const encodedTitle = encodeURIComponent(chat.title)
|
const encodedTitle = encodeURIComponent(chat.title)
|
||||||
return `v${chat.schemaVersion}_${encodedTitle}_${chat.updatedAt}_${chat.id}.json`
|
const workspaceId = chat.workspace || 'vault'
|
||||||
|
return `v${chat.schemaVersion}_${encodedTitle}_${chat.updatedAt}_${chat.id}_${workspaceId}.json`
|
||||||
}
|
}
|
||||||
|
|
||||||
protected parseFileName(fileName: string): ChatConversationMeta | null {
|
protected parseFileName(fileName: string): ChatConversationMeta | null {
|
||||||
// Parse: v{schemaVersion}_{title}_{updatedAt}_{id}.json
|
// 使用一个正则表达式,工作区部分为可选: v{schemaVersion}_{title}_{updatedAt}_{id}_{workspaceId}?.json
|
||||||
const regex = new RegExp(
|
const regex = new RegExp(
|
||||||
`^v${CHAT_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)\\.json$`,
|
`^v${CHAT_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)(?:_([^_]+))?\\.json$`,
|
||||||
)
|
)
|
||||||
const match = fileName.match(regex)
|
const match = fileName.match(regex)
|
||||||
|
|
||||||
if (!match) return null
|
if (!match) return null
|
||||||
|
|
||||||
const title = decodeURIComponent(match[1])
|
const title = decodeURIComponent(match[1])
|
||||||
const updatedAt = parseInt(match[2], 10)
|
const updatedAt = parseInt(match[2], 10)
|
||||||
const id = match[3]
|
const id = match[3]
|
||||||
|
const workspaceId = match[4] // 可能为undefined(老格式)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id,
|
id,
|
||||||
@ -44,6 +52,8 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
title,
|
title,
|
||||||
updatedAt,
|
updatedAt,
|
||||||
createdAt: 0,
|
createdAt: 0,
|
||||||
|
// 如果没有工作区信息(老格式),则认为是vault(全局消息)
|
||||||
|
workspace: workspaceId === 'vault' ? undefined : workspaceId,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -66,6 +76,20 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
}
|
}
|
||||||
|
|
||||||
await this.create(newChat)
|
await this.create(newChat)
|
||||||
|
|
||||||
|
// 如果有工作区信息,添加到工作区的聊天历史中
|
||||||
|
if (newChat.workspace && this.workspaceManager) {
|
||||||
|
try {
|
||||||
|
await this.workspaceManager.addChatToWorkspace(
|
||||||
|
newChat.workspace,
|
||||||
|
newChat.id,
|
||||||
|
newChat.title
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to add chat to workspace:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return newChat
|
return newChat
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -102,6 +126,23 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
}
|
}
|
||||||
|
|
||||||
await this.update(chat, updatedChat)
|
await this.update(chat, updatedChat)
|
||||||
|
|
||||||
|
// 如果标题或工作区发生变化,更新工作区的聊天历史
|
||||||
|
if (this.workspaceManager && (updates.title !== undefined || updates.workspace !== undefined)) {
|
||||||
|
const workspaceId = updatedChat.workspace || chat.workspace
|
||||||
|
if (workspaceId) {
|
||||||
|
try {
|
||||||
|
await this.workspaceManager.addChatToWorkspace(
|
||||||
|
workspaceId,
|
||||||
|
updatedChat.id,
|
||||||
|
updatedChat.title
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to update chat in workspace:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return updatedChat
|
return updatedChat
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -111,9 +152,22 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
|
|
||||||
if (targetsToDelete.length === 0) return false
|
if (targetsToDelete.length === 0) return false
|
||||||
|
|
||||||
|
// 获取聊天的工作区信息(从第一个匹配的元数据中获取)
|
||||||
|
const chatToDelete = await this.findById(id)
|
||||||
|
const workspaceId = chatToDelete?.workspace
|
||||||
|
|
||||||
// Delete all files associated with this ID
|
// Delete all files associated with this ID
|
||||||
await Promise.all(targetsToDelete.map(meta => this.delete(meta.fileName)))
|
await Promise.all(targetsToDelete.map(meta => this.delete(meta.fileName)))
|
||||||
|
|
||||||
|
// 从工作区的聊天历史中移除
|
||||||
|
if (workspaceId && this.workspaceManager) {
|
||||||
|
try {
|
||||||
|
await this.workspaceManager.removeChatFromWorkspace(workspaceId, id)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to remove chat from workspace:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -126,7 +180,10 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
if (!chatsById.has(meta.id)) {
|
if (!chatsById.has(meta.id)) {
|
||||||
chatsById.set(meta.id, [])
|
chatsById.set(meta.id, [])
|
||||||
}
|
}
|
||||||
chatsById.get(meta.id)!.push(meta)
|
const chatGroup = chatsById.get(meta.id)
|
||||||
|
if (chatGroup) {
|
||||||
|
chatGroup.push(meta)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const filesToDelete: string[] = []
|
const filesToDelete: string[] = []
|
||||||
@ -151,11 +208,12 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
return filesToDelete.length
|
return filesToDelete.length
|
||||||
}
|
}
|
||||||
|
|
||||||
public async listChats(): Promise<ChatConversationMeta[]> {
|
public async listChats(workspaceFilter?: string): Promise<ChatConversationMeta[]> {
|
||||||
|
console.log('listChats', workspaceFilter)
|
||||||
const metadata = await this.listMetadata()
|
const metadata = await this.listMetadata()
|
||||||
|
|
||||||
// Use a Map to store the latest version of each chat by ID.
|
// Use a Map to store the latest version of each chat by ID.
|
||||||
const latestChats = new Map<string, ChatConversationMeta>()
|
const latestChats = new Map<string, ChatConversationMeta & { fileName: string }>()
|
||||||
|
|
||||||
for (const meta of metadata) {
|
for (const meta of metadata) {
|
||||||
const existing = latestChats.get(meta.id)
|
const existing = latestChats.get(meta.id)
|
||||||
@ -165,7 +223,27 @@ export class ChatManager extends AbstractJsonRepository<
|
|||||||
}
|
}
|
||||||
|
|
||||||
const uniqueMetadata = Array.from(latestChats.values())
|
const uniqueMetadata = Array.from(latestChats.values())
|
||||||
const sorted = uniqueMetadata.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
||||||
|
// 将metadata转换为ChatConversationMeta格式
|
||||||
|
const chatMetadata: ChatConversationMeta[] = uniqueMetadata.map((meta) => ({
|
||||||
|
id: meta.id,
|
||||||
|
schemaVersion: meta.schemaVersion,
|
||||||
|
title: meta.title,
|
||||||
|
updatedAt: meta.updatedAt,
|
||||||
|
createdAt: meta.createdAt,
|
||||||
|
workspace: meta.workspace
|
||||||
|
}))
|
||||||
|
|
||||||
|
// 如果指定了工作区过滤器,则过滤对话
|
||||||
|
let filteredMetadata = chatMetadata
|
||||||
|
if (workspaceFilter !== undefined && workspaceFilter !== 'vault') {
|
||||||
|
// 获取指定工作区的对话
|
||||||
|
filteredMetadata = chatMetadata.filter(meta =>
|
||||||
|
meta.workspace === workspaceFilter
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const sorted = filteredMetadata.sort((a, b) => b.updatedAt - a.updatedAt)
|
||||||
return sorted
|
return sorted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,6 +9,7 @@ export type ChatConversation = {
|
|||||||
createdAt: number
|
createdAt: number
|
||||||
updatedAt: number
|
updatedAt: number
|
||||||
schemaVersion: number
|
schemaVersion: number
|
||||||
|
workspace?: string // 工作区ID,可选字段用于向后兼容
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ChatConversationMetadata = {
|
export type ChatConversationMetadata = {
|
||||||
@ -16,4 +17,5 @@ export type ChatConversationMetadata = {
|
|||||||
title: string
|
title: string
|
||||||
updatedAt: number
|
updatedAt: number
|
||||||
schemaVersion: number
|
schemaVersion: number
|
||||||
|
workspace?: string // 工作区ID,可选字段用于向后兼容
|
||||||
}
|
}
|
||||||
|
|||||||
@ -118,11 +118,11 @@ export class WorkspaceManager extends AbstractJsonRepository<
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async addChatToWorkspace(
|
public async addChatToWorkspace(
|
||||||
workspaceId: string,
|
workspaceName: string,
|
||||||
chatId: string,
|
chatId: string,
|
||||||
chatTitle: string
|
chatTitle: string
|
||||||
): Promise<Workspace | null> {
|
): Promise<Workspace | null> {
|
||||||
const workspace = await this.findById(workspaceId)
|
const workspace = await this.findByName(workspaceName)
|
||||||
if (!workspace) return null
|
if (!workspace) return null
|
||||||
|
|
||||||
const existingChatIndex = workspace.chatHistory.findIndex(
|
const existingChatIndex = workspace.chatHistory.findIndex(
|
||||||
@ -137,7 +137,7 @@ export class WorkspaceManager extends AbstractJsonRepository<
|
|||||||
workspace.chatHistory.push({ id: chatId, title: chatTitle })
|
workspace.chatHistory.push({ id: chatId, title: chatTitle })
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.updateWorkspace(workspaceId, {
|
return this.updateWorkspace(workspace.id, {
|
||||||
chatHistory: workspace.chatHistory
|
chatHistory: workspace.chatHistory
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
274
src/embedworker/EmbeddingManager.ts
Normal file
274
src/embedworker/EmbeddingManager.ts
Normal file
@ -0,0 +1,274 @@
|
|||||||
|
// 导入完整的嵌入 Worker
|
||||||
|
// @ts-nocheck
|
||||||
|
import EmbedWorker from './embed.worker';
|
||||||
|
|
||||||
|
// 类型定义
|
||||||
|
export interface EmbedResult {
|
||||||
|
vec: number[];
|
||||||
|
tokens: number;
|
||||||
|
embed_input?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ModelLoadResult {
|
||||||
|
model_loaded: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ModelUnloadResult {
|
||||||
|
model_unloaded: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TokenCountResult {
|
||||||
|
tokens: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class EmbeddingManager {
|
||||||
|
private worker: Worker;
|
||||||
|
private requests = new Map<number, { resolve: (value: any) => void; reject: (reason?: any) => void }>();
|
||||||
|
private nextRequestId = 0;
|
||||||
|
private isModelLoaded = false;
|
||||||
|
private currentModelId: string | null = null;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
// 创建 Worker,使用与 pgworker 相同的模式
|
||||||
|
this.worker = new EmbedWorker();
|
||||||
|
|
||||||
|
// 统一监听来自 Worker 的所有消息
|
||||||
|
this.worker.onmessage = (event) => {
|
||||||
|
const { id, result, error } = event.data;
|
||||||
|
|
||||||
|
// 根据返回的 id 找到对应的 Promise 回调
|
||||||
|
const request = this.requests.get(id);
|
||||||
|
|
||||||
|
if (request) {
|
||||||
|
if (error) {
|
||||||
|
request.reject(new Error(error));
|
||||||
|
} else {
|
||||||
|
request.resolve(result);
|
||||||
|
}
|
||||||
|
// 完成后从 Map 中删除
|
||||||
|
this.requests.delete(id);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.worker.onerror = (error) => {
|
||||||
|
console.error("EmbeddingWorker error:", error);
|
||||||
|
// 拒绝所有待处理的请求
|
||||||
|
this.requests.forEach(request => {
|
||||||
|
request.reject(error);
|
||||||
|
});
|
||||||
|
this.requests.clear();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 向 Worker 发送一个请求,并返回一个 Promise,该 Promise 将在收到响应时解析。
|
||||||
|
* @param method 要调用的方法 (e.g., 'load', 'embed_batch')
|
||||||
|
* @param params 方法所需的参数
|
||||||
|
*/
|
||||||
|
private postRequest<T>(method: string, params: any): Promise<T> {
|
||||||
|
return new Promise<T>((resolve, reject) => {
|
||||||
|
const id = this.nextRequestId++;
|
||||||
|
this.requests.set(id, { resolve, reject });
|
||||||
|
this.worker.postMessage({ method, params, id });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 加载指定的嵌入模型到 Worker 中。
|
||||||
|
* @param modelId 模型ID, 例如 'TaylorAI/bge-micro-v2'
|
||||||
|
* @param useGpu 是否使用GPU加速,默认为false
|
||||||
|
*/
|
||||||
|
public async loadModel(modelId: string, useGpu: boolean = false): Promise<ModelLoadResult> {
|
||||||
|
console.log(`Loading embedding model: ${modelId}, GPU: ${useGpu}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 如果已经加载了相同的模型,直接返回
|
||||||
|
if (this.isModelLoaded && this.currentModelId === modelId) {
|
||||||
|
console.log(`Model ${modelId} already loaded`);
|
||||||
|
return { model_loaded: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果加载了不同的模型,先卸载
|
||||||
|
if (this.isModelLoaded && this.currentModelId !== modelId) {
|
||||||
|
console.log(`Unloading previous model: ${this.currentModelId}`);
|
||||||
|
await this.unloadModel();
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.postRequest<ModelLoadResult>('load', {
|
||||||
|
model_key: modelId,
|
||||||
|
use_gpu: useGpu
|
||||||
|
});
|
||||||
|
|
||||||
|
this.isModelLoaded = result.model_loaded;
|
||||||
|
this.currentModelId = result.model_loaded ? modelId : null;
|
||||||
|
|
||||||
|
if (result.model_loaded) {
|
||||||
|
console.log(`Model ${modelId} loaded successfully`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to load model ${modelId}:`, error);
|
||||||
|
this.isModelLoaded = false;
|
||||||
|
this.currentModelId = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 为一批文本生成嵌入向量。
|
||||||
|
* @param texts 要处理的文本数组
|
||||||
|
* @returns 返回一个包含向量和 token 信息的对象数组
|
||||||
|
*/
|
||||||
|
public async embedBatch(texts: string[]): Promise<EmbedResult[]> {
|
||||||
|
if (!this.isModelLoaded) {
|
||||||
|
throw new Error('Model not loaded. Please call loadModel() first.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!texts || texts.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Generating embeddings for ${texts.length} texts`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const inputs = texts.map(text => ({ embed_input: text }));
|
||||||
|
const results = await this.postRequest<EmbedResult[]>('embed_batch', { inputs });
|
||||||
|
|
||||||
|
console.log(`Generated ${results.length} embeddings`);
|
||||||
|
return results;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to generate embeddings:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 为单个文本生成嵌入向量。
|
||||||
|
* @param text 要处理的文本
|
||||||
|
* @returns 返回包含向量和 token 信息的对象
|
||||||
|
*/
|
||||||
|
public async embed(text: string): Promise<EmbedResult> {
|
||||||
|
if (!text || text.trim().length === 0) {
|
||||||
|
throw new Error('Text cannot be empty');
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = await this.embedBatch([text]);
|
||||||
|
if (results.length === 0) {
|
||||||
|
throw new Error('Failed to generate embedding');
|
||||||
|
}
|
||||||
|
|
||||||
|
return results[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 计算文本的 token 数量。
|
||||||
|
* @param text 要计算的文本
|
||||||
|
*/
|
||||||
|
public async countTokens(text: string): Promise<TokenCountResult> {
|
||||||
|
if (!this.isModelLoaded) {
|
||||||
|
throw new Error('Model not loaded. Please call loadModel() first.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!text) {
|
||||||
|
return { tokens: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await this.postRequest<TokenCountResult>('count_tokens', text);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to count tokens:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 卸载模型,释放内存。
|
||||||
|
*/
|
||||||
|
public async unloadModel(): Promise<ModelUnloadResult> {
|
||||||
|
if (!this.isModelLoaded) {
|
||||||
|
console.log('No model to unload');
|
||||||
|
return { model_unloaded: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log(`Unloading model: ${this.currentModelId}`);
|
||||||
|
const result = await this.postRequest<ModelUnloadResult>('unload', {});
|
||||||
|
|
||||||
|
this.isModelLoaded = false;
|
||||||
|
this.currentModelId = null;
|
||||||
|
|
||||||
|
console.log('Model unloaded successfully');
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to unload model:', error);
|
||||||
|
// 即使卸载失败,也重置状态
|
||||||
|
this.isModelLoaded = false;
|
||||||
|
this.currentModelId = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 检查模型是否已加载。
|
||||||
|
*/
|
||||||
|
public get modelLoaded(): boolean {
|
||||||
|
return this.isModelLoaded;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前加载的模型ID。
|
||||||
|
*/
|
||||||
|
public get currentModel(): string | null {
|
||||||
|
return this.currentModelId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取支持的模型列表。
|
||||||
|
*/
|
||||||
|
public getSupportedModels(): string[] {
|
||||||
|
return [
|
||||||
|
'Xenova/all-MiniLM-L6-v2',
|
||||||
|
'Xenova/bge-small-en-v1.5',
|
||||||
|
'Xenova/bge-base-en-v1.5',
|
||||||
|
'Xenova/jina-embeddings-v2-base-zh',
|
||||||
|
'Xenova/jina-embeddings-v2-small-en',
|
||||||
|
'Xenova/multilingual-e5-small',
|
||||||
|
'Xenova/multilingual-e5-base',
|
||||||
|
'Xenova/gte-small',
|
||||||
|
'Xenova/e5-small-v2',
|
||||||
|
'Xenova/e5-base-v2'
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取模型信息。
|
||||||
|
*/
|
||||||
|
public getModelInfo(modelId: string): { dims: number; maxTokens: number; description: string } | null {
|
||||||
|
const modelInfoMap: Record<string, { dims: number; maxTokens: number; description: string }> = {
|
||||||
|
'Xenova/all-MiniLM-L6-v2': { dims: 384, maxTokens: 512, description: 'All-MiniLM-L6-v2 (推荐,轻量级)' },
|
||||||
|
'Xenova/bge-small-en-v1.5': { dims: 384, maxTokens: 512, description: 'BGE-small-en-v1.5' },
|
||||||
|
'Xenova/bge-base-en-v1.5': { dims: 768, maxTokens: 512, description: 'BGE-base-en-v1.5 (更高质量)' },
|
||||||
|
'Xenova/jina-embeddings-v2-base-zh': { dims: 768, maxTokens: 8192, description: 'Jina-v2-base-zh (中英双语)' },
|
||||||
|
'Xenova/jina-embeddings-v2-small-en': { dims: 512, maxTokens: 8192, description: 'Jina-v2-small-en' },
|
||||||
|
'Xenova/multilingual-e5-small': { dims: 384, maxTokens: 512, description: 'E5-small (多语言)' },
|
||||||
|
'Xenova/multilingual-e5-base': { dims: 768, maxTokens: 512, description: 'E5-base (多语言,更高质量)' },
|
||||||
|
'Xenova/gte-small': { dims: 384, maxTokens: 512, description: 'GTE-small' },
|
||||||
|
'Xenova/e5-small-v2': { dims: 384, maxTokens: 512, description: 'E5-small-v2' },
|
||||||
|
'Xenova/e5-base-v2': { dims: 768, maxTokens: 512, description: 'E5-base-v2 (更高质量)' }
|
||||||
|
};
|
||||||
|
|
||||||
|
return modelInfoMap[modelId] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 终止 Worker,释放资源。
|
||||||
|
*/
|
||||||
|
public terminate() {
|
||||||
|
this.worker.terminate();
|
||||||
|
this.requests.clear();
|
||||||
|
this.isModelLoaded = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
171
src/embedworker/README.md
Normal file
171
src/embedworker/README.md
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
# 本地嵌入功能
|
||||||
|
|
||||||
|
这个模块提供了在 Web Worker 中运行的本地嵌入功能,使用 Transformers.js 库来生成文本的向量表示。
|
||||||
|
|
||||||
|
## 功能特性
|
||||||
|
|
||||||
|
- 🚀 **高性能**: 在 Web Worker 中运行,不阻塞主线程
|
||||||
|
- 🔒 **隐私保护**: 完全本地运行,数据不离开设备
|
||||||
|
- 🎯 **多模型支持**: 支持多种预训练的嵌入模型
|
||||||
|
- 💾 **内存管理**: 自动管理模型加载和卸载
|
||||||
|
- 🔧 **类型安全**: 完整的 TypeScript 类型支持
|
||||||
|
|
||||||
|
## 快速开始
|
||||||
|
|
||||||
|
### 基本使用
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { embeddingManager } from './embedworker';
|
||||||
|
|
||||||
|
// 加载模型
|
||||||
|
await embeddingManager.loadModel('Xenova/all-MiniLM-L6-v2');
|
||||||
|
|
||||||
|
// 生成单个文本的嵌入向量
|
||||||
|
const result = await embeddingManager.embed('Hello, world!');
|
||||||
|
console.log(result.vec); // [0.1234, -0.5678, ...]
|
||||||
|
console.log(result.tokens); // 3
|
||||||
|
|
||||||
|
// 批量生成嵌入向量
|
||||||
|
const texts = ['Hello', 'World', 'AI is amazing'];
|
||||||
|
const results = await embeddingManager.embedBatch(texts);
|
||||||
|
|
||||||
|
// 计算 token 数量
|
||||||
|
const tokenCount = await embeddingManager.countTokens('How many tokens?');
|
||||||
|
console.log(tokenCount.tokens); // 4
|
||||||
|
```
|
||||||
|
|
||||||
|
### 高级使用
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { EmbeddingManager } from './embedworker';
|
||||||
|
|
||||||
|
// 创建自定义实例
|
||||||
|
const customEmbedding = new EmbeddingManager();
|
||||||
|
|
||||||
|
// 使用 GPU 加速(如果支持)
|
||||||
|
await customEmbedding.loadModel('Xenova/all-MiniLM-L6-v2', true);
|
||||||
|
|
||||||
|
// 检查模型状态
|
||||||
|
console.log(customEmbedding.modelLoaded); // true
|
||||||
|
console.log(customEmbedding.currentModel); // 'TaylorAI/bge-micro-v2'
|
||||||
|
|
||||||
|
// 获取支持的模型列表
|
||||||
|
const models = customEmbedding.getSupportedModels();
|
||||||
|
console.log(models);
|
||||||
|
|
||||||
|
// 获取模型信息
|
||||||
|
const modelInfo = customEmbedding.getModelInfo('Xenova/all-MiniLM-L6-v2');
|
||||||
|
console.log(modelInfo); // { dims: 384, maxTokens: 512, description: '...' }
|
||||||
|
|
||||||
|
// 切换模型
|
||||||
|
await customEmbedding.loadModel('Snowflake/snowflake-arctic-embed-xs');
|
||||||
|
|
||||||
|
// 清理资源
|
||||||
|
await customEmbedding.unloadModel();
|
||||||
|
customEmbedding.terminate();
|
||||||
|
```
|
||||||
|
|
||||||
|
## 支持的模型
|
||||||
|
|
||||||
|
| 模型 | 维度 | 最大Token | 描述 |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| Xenova/all-MiniLM-L6-v2 | 384 | 512 | All-MiniLM-L6-v2 (推荐,轻量级) |
|
||||||
|
| Xenova/bge-small-en-v1.5 | 384 | 512 | BGE-small-en-v1.5 |
|
||||||
|
| Xenova/bge-base-en-v1.5 | 768 | 512 | BGE-base-en-v1.5 (更高质量) |
|
||||||
|
| Xenova/jina-embeddings-v2-base-zh | 768 | 8192 | Jina-v2-base-zh (中英双语) |
|
||||||
|
| Xenova/jina-embeddings-v2-small-en | 512 | 8192 | Jina-v2-small-en |
|
||||||
|
| Xenova/multilingual-e5-small | 384 | 512 | E5-small (多语言) |
|
||||||
|
| Xenova/multilingual-e5-base | 768 | 512 | E5-base (多语言,更高质量) |
|
||||||
|
| Xenova/gte-small | 384 | 512 | GTE-small |
|
||||||
|
| Xenova/e5-small-v2 | 384 | 512 | E5-small-v2 |
|
||||||
|
| Xenova/e5-base-v2 | 768 | 512 | E5-base-v2 (更高质量) |
|
||||||
|
|
||||||
|
## API 参考
|
||||||
|
|
||||||
|
### EmbeddingManager
|
||||||
|
|
||||||
|
#### 方法
|
||||||
|
|
||||||
|
- `loadModel(modelId: string, useGpu?: boolean): Promise<ModelLoadResult>`
|
||||||
|
- 加载指定的嵌入模型
|
||||||
|
- `modelId`: 模型标识符
|
||||||
|
- `useGpu`: 是否使用 GPU 加速(默认 false)
|
||||||
|
|
||||||
|
- `embed(text: string): Promise<EmbedResult>`
|
||||||
|
- 为单个文本生成嵌入向量
|
||||||
|
- 返回包含向量和 token 数量的结果
|
||||||
|
|
||||||
|
- `embedBatch(texts: string[]): Promise<EmbedResult[]>`
|
||||||
|
- 为多个文本批量生成嵌入向量
|
||||||
|
- 更高效的批处理方式
|
||||||
|
|
||||||
|
- `countTokens(text: string): Promise<TokenCountResult>`
|
||||||
|
- 计算文本的 token 数量
|
||||||
|
|
||||||
|
- `unloadModel(): Promise<ModelUnloadResult>`
|
||||||
|
- 卸载当前模型,释放内存
|
||||||
|
|
||||||
|
- `terminate(): void`
|
||||||
|
- 终止 Worker,释放所有资源
|
||||||
|
|
||||||
|
#### 属性
|
||||||
|
|
||||||
|
- `modelLoaded: boolean` - 模型是否已加载
|
||||||
|
- `currentModel: string | null` - 当前加载的模型ID
|
||||||
|
|
||||||
|
#### 工具方法
|
||||||
|
|
||||||
|
- `getSupportedModels(): string[]` - 获取支持的模型列表
|
||||||
|
- `getModelInfo(modelId: string)` - 获取模型详细信息
|
||||||
|
|
||||||
|
### 类型定义
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface EmbedResult {
|
||||||
|
vec: number[]; // 嵌入向量
|
||||||
|
tokens: number; // token 数量
|
||||||
|
embed_input?: string; // 原始输入文本
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ModelLoadResult {
|
||||||
|
model_loaded: boolean; // 是否加载成功
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ModelUnloadResult {
|
||||||
|
model_unloaded: boolean; // 是否卸载成功
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TokenCountResult {
|
||||||
|
tokens: number; // token 数量
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 错误处理
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
try {
|
||||||
|
await embeddingManager.loadModel('invalid-model');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('加载模型失败:', error.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await embeddingManager.embed('');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('文本不能为空:', error.message);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 性能考虑
|
||||||
|
|
||||||
|
1. **模型加载**: 首次加载模型需要下载和初始化,可能需要几秒到几分钟
|
||||||
|
2. **批处理**: 使用 `embedBatch` 比多次调用 `embed` 更高效
|
||||||
|
3. **内存使用**: 大模型需要更多内存,注意设备限制
|
||||||
|
4. **GPU 加速**: 在支持 WebGPU 的浏览器中可以启用 GPU 加速
|
||||||
|
|
||||||
|
## 注意事项
|
||||||
|
|
||||||
|
- 首次使用某个模型时需要从 Hugging Face 下载,请确保网络连接正常
|
||||||
|
- 模型文件会被浏览器缓存,后续使用会更快
|
||||||
|
- 在移动设备上使用大模型可能会遇到内存限制
|
||||||
|
- Worker 在后台运行,不会阻塞 UI 线程
|
||||||
353
src/embedworker/embed.worker.ts
Normal file
353
src/embedworker/embed.worker.ts
Normal file
@ -0,0 +1,353 @@
|
|||||||
|
// 完整的嵌入 Worker,使用 Transformers.js
|
||||||
|
console.log('Embedding worker loaded');
|
||||||
|
|
||||||
|
// 类型定义
|
||||||
|
interface EmbedInput {
|
||||||
|
embed_input: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EmbedResult {
|
||||||
|
vec: number[];
|
||||||
|
tokens: number;
|
||||||
|
embed_input?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WorkerMessage {
|
||||||
|
method: string;
|
||||||
|
params: any;
|
||||||
|
id: number;
|
||||||
|
worker_id?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WorkerResponse {
|
||||||
|
id: number;
|
||||||
|
result?: any;
|
||||||
|
error?: string;
|
||||||
|
worker_id?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 全局变量
|
||||||
|
let model: any = null;
|
||||||
|
let pipeline: any = null;
|
||||||
|
let tokenizer: any = null;
|
||||||
|
let processing_message = false;
|
||||||
|
let transformersLoaded = false;
|
||||||
|
|
||||||
|
// 动态导入 Transformers.js
|
||||||
|
async function loadTransformers() {
|
||||||
|
if (transformersLoaded) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log('Loading Transformers.js...');
|
||||||
|
|
||||||
|
// 尝试使用旧版本的 Transformers.js,它在 Worker 中更稳定
|
||||||
|
const { pipeline: pipelineFactory, env, AutoTokenizer } = await import('@xenova/transformers');
|
||||||
|
|
||||||
|
// 配置环境以适应浏览器 Worker
|
||||||
|
env.allowLocalModels = false;
|
||||||
|
env.allowRemoteModels = true;
|
||||||
|
|
||||||
|
// 配置 WASM 后端
|
||||||
|
env.backends.onnx.wasm.numThreads = 2; // 在 Worker 中使用单线程
|
||||||
|
env.backends.onnx.wasm.simd = true;
|
||||||
|
|
||||||
|
// 禁用 Node.js 特定功能
|
||||||
|
env.useFS = false;
|
||||||
|
env.useBrowserCache = true;
|
||||||
|
|
||||||
|
// 存储导入的函数
|
||||||
|
(globalThis as any).pipelineFactory = pipelineFactory;
|
||||||
|
(globalThis as any).AutoTokenizer = AutoTokenizer;
|
||||||
|
(globalThis as any).env = env;
|
||||||
|
|
||||||
|
transformersLoaded = true;
|
||||||
|
console.log('Transformers.js loaded successfully');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load Transformers.js:', error);
|
||||||
|
throw new Error(`Failed to load Transformers.js: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 加载模型
|
||||||
|
async function loadModel(modelKey: string, useGpu: boolean = false) {
|
||||||
|
try {
|
||||||
|
console.log(`Loading model: ${modelKey}, GPU: ${useGpu}`);
|
||||||
|
|
||||||
|
// 确保 Transformers.js 已加载
|
||||||
|
await loadTransformers();
|
||||||
|
|
||||||
|
const pipelineFactory = (globalThis as any).pipelineFactory;
|
||||||
|
const AutoTokenizer = (globalThis as any).AutoTokenizer;
|
||||||
|
const env = (globalThis as any).env;
|
||||||
|
|
||||||
|
// 配置管道选项
|
||||||
|
const pipelineOpts: any = {
|
||||||
|
quantized: true,
|
||||||
|
progress_callback: (progress: any) => {
|
||||||
|
console.log('Model loading progress:', progress);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (useGpu && typeof navigator !== 'undefined' && 'gpu' in navigator) {
|
||||||
|
console.log('[Transformers] Attempting to use GPU');
|
||||||
|
try {
|
||||||
|
pipelineOpts.device = 'webgpu';
|
||||||
|
pipelineOpts.dtype = 'fp32';
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('[Transformers] GPU not available, falling back to CPU');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log('[Transformers] Using CPU');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 创建嵌入管道
|
||||||
|
pipeline = await pipelineFactory('feature-extraction', modelKey, pipelineOpts);
|
||||||
|
|
||||||
|
// 创建分词器
|
||||||
|
tokenizer = await AutoTokenizer.from_pretrained(modelKey);
|
||||||
|
|
||||||
|
model = {
|
||||||
|
loaded: true,
|
||||||
|
model_key: modelKey,
|
||||||
|
use_gpu: useGpu
|
||||||
|
};
|
||||||
|
|
||||||
|
console.log(`Model ${modelKey} loaded successfully`);
|
||||||
|
return { model_loaded: true };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading model:', error);
|
||||||
|
throw new Error(`Failed to load model: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 卸载模型
|
||||||
|
async function unloadModel() {
|
||||||
|
try {
|
||||||
|
console.log('Unloading model...');
|
||||||
|
|
||||||
|
if (pipeline) {
|
||||||
|
if (pipeline.destroy) {
|
||||||
|
pipeline.destroy();
|
||||||
|
}
|
||||||
|
pipeline = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tokenizer) {
|
||||||
|
tokenizer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
model = null;
|
||||||
|
|
||||||
|
console.log('Model unloaded successfully');
|
||||||
|
return { model_unloaded: true };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error unloading model:', error);
|
||||||
|
throw new Error(`Failed to unload model: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 计算 token 数量
|
||||||
|
async function countTokens(input: string) {
|
||||||
|
try {
|
||||||
|
if (!tokenizer) {
|
||||||
|
throw new Error('Tokenizer not loaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { input_ids } = await tokenizer(input);
|
||||||
|
return { tokens: input_ids.data.length };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error counting tokens:', error);
|
||||||
|
throw new Error(`Failed to count tokens: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 生成嵌入向量
|
||||||
|
async function embedBatch(inputs: EmbedInput[]): Promise<EmbedResult[]> {
|
||||||
|
try {
|
||||||
|
if (!pipeline || !tokenizer) {
|
||||||
|
throw new Error('Model not loaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Processing ${inputs.length} inputs`);
|
||||||
|
|
||||||
|
// 过滤空输入
|
||||||
|
const filteredInputs = inputs.filter(item => item.embed_input && item.embed_input.length > 0);
|
||||||
|
|
||||||
|
if (filteredInputs.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// 批处理大小(可以根据需要调整)
|
||||||
|
const batchSize = 1;
|
||||||
|
|
||||||
|
if (filteredInputs.length > batchSize) {
|
||||||
|
console.log(`Processing ${filteredInputs.length} inputs in batches of ${batchSize}`);
|
||||||
|
const results: EmbedResult[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < filteredInputs.length; i += batchSize) {
|
||||||
|
const batch = filteredInputs.slice(i, i + batchSize);
|
||||||
|
const batchResults = await processBatch(batch);
|
||||||
|
results.push(...batchResults);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await processBatch(filteredInputs);
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error in embed batch:', error);
|
||||||
|
throw new Error(`Failed to generate embeddings: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理单个批次
|
||||||
|
async function processBatch(batchInputs: EmbedInput[]): Promise<EmbedResult[]> {
|
||||||
|
try {
|
||||||
|
// 计算每个输入的 token 数量
|
||||||
|
const tokens = await Promise.all(
|
||||||
|
batchInputs.map(item => countTokens(item.embed_input))
|
||||||
|
);
|
||||||
|
|
||||||
|
// 准备嵌入输入(处理超长文本)
|
||||||
|
const maxTokens = 512; // 大多数模型的最大 token 限制
|
||||||
|
const embedInputs = await Promise.all(
|
||||||
|
batchInputs.map(async (item, i) => {
|
||||||
|
if (tokens[i].tokens < maxTokens) {
|
||||||
|
return item.embed_input;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 截断超长文本
|
||||||
|
let tokenCt = tokens[i].tokens;
|
||||||
|
let truncatedInput = item.embed_input;
|
||||||
|
|
||||||
|
while (tokenCt > maxTokens) {
|
||||||
|
const pct = maxTokens / tokenCt;
|
||||||
|
const maxChars = Math.floor(truncatedInput.length * pct * 0.9);
|
||||||
|
truncatedInput = truncatedInput.substring(0, maxChars) + '...';
|
||||||
|
tokenCt = (await countTokens(truncatedInput)).tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
tokens[i].tokens = tokenCt;
|
||||||
|
return truncatedInput;
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// 生成嵌入向量
|
||||||
|
const resp = await pipeline(embedInputs, { pooling: 'mean', normalize: true });
|
||||||
|
|
||||||
|
// 处理结果
|
||||||
|
return batchInputs.map((item, i) => ({
|
||||||
|
vec: Array.from(resp[i].data).map((val: number) => Math.round(val * 1e8) / 1e8),
|
||||||
|
tokens: tokens[i].tokens,
|
||||||
|
embed_input: item.embed_input
|
||||||
|
}));
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error processing batch:', error);
|
||||||
|
|
||||||
|
// 如果批处理失败,尝试逐个处理
|
||||||
|
return Promise.all(
|
||||||
|
batchInputs.map(async (item) => {
|
||||||
|
try {
|
||||||
|
const result = await pipeline(item.embed_input, { pooling: 'mean', normalize: true });
|
||||||
|
const tokenCount = await countTokens(item.embed_input);
|
||||||
|
|
||||||
|
return {
|
||||||
|
vec: Array.from(result[0].data).map((val: number) => Math.round(val * 1e8) / 1e8),
|
||||||
|
tokens: tokenCount.tokens,
|
||||||
|
embed_input: item.embed_input
|
||||||
|
};
|
||||||
|
} catch (singleError) {
|
||||||
|
console.error('Error processing single item:', singleError);
|
||||||
|
return {
|
||||||
|
vec: [],
|
||||||
|
tokens: 0,
|
||||||
|
embed_input: item.embed_input,
|
||||||
|
error: (singleError as Error).message
|
||||||
|
} as any;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理消息
|
||||||
|
async function processMessage(data: WorkerMessage): Promise<WorkerResponse> {
|
||||||
|
const { method, params, id, worker_id } = data;
|
||||||
|
|
||||||
|
try {
|
||||||
|
let result: any;
|
||||||
|
|
||||||
|
switch (method) {
|
||||||
|
case 'load':
|
||||||
|
console.log('Load method called with params:', params);
|
||||||
|
result = await loadModel(params.model_key, params.use_gpu || false);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'unload':
|
||||||
|
console.log('Unload method called');
|
||||||
|
result = await unloadModel();
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'embed_batch':
|
||||||
|
console.log('Embed batch method called');
|
||||||
|
if (!model) {
|
||||||
|
throw new Error('Model not loaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待之前的处理完成
|
||||||
|
if (processing_message) {
|
||||||
|
while (processing_message) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 100));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processing_message = true;
|
||||||
|
result = await embedBatch(params.inputs);
|
||||||
|
processing_message = false;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'count_tokens':
|
||||||
|
console.log('Count tokens method called');
|
||||||
|
if (!model) {
|
||||||
|
throw new Error('Model not loaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待之前的处理完成
|
||||||
|
if (processing_message) {
|
||||||
|
while (processing_message) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 100));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processing_message = true;
|
||||||
|
result = await countTokens(params);
|
||||||
|
processing_message = false;
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new Error(`Unknown method: ${method}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { id, result, worker_id };
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error processing message:', error);
|
||||||
|
processing_message = false;
|
||||||
|
return { id, error: (error as Error).message, worker_id };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 监听消息
|
||||||
|
self.addEventListener('message', async (event) => {
|
||||||
|
console.log('Worker received message:', event.data);
|
||||||
|
const response = await processMessage(event.data);
|
||||||
|
console.log('Worker sending response:', response);
|
||||||
|
self.postMessage(response);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Embedding worker ready');
|
||||||
15
src/embedworker/index.ts
Normal file
15
src/embedworker/index.ts
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
import { EmbeddingManager } from "./EmbeddingManager";
|
||||||
|
|
||||||
|
// 创建一个单例的 Manager,以便在整个应用中共享同一个 Worker
|
||||||
|
export const embeddingManager = new EmbeddingManager();
|
||||||
|
|
||||||
|
// 导出 EmbeddingManager 类以便其他地方使用
|
||||||
|
export { EmbeddingManager };
|
||||||
|
|
||||||
|
// 导出类型定义
|
||||||
|
export type {
|
||||||
|
EmbedResult,
|
||||||
|
ModelLoadResult,
|
||||||
|
ModelUnloadResult,
|
||||||
|
TokenCountResult
|
||||||
|
} from './EmbeddingManager';
|
||||||
@ -4,8 +4,10 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
|
|||||||
|
|
||||||
import { editorStateToPlainText } from '../components/chat-view/chat-input/utils/editor-state-to-plain-text'
|
import { editorStateToPlainText } from '../components/chat-view/chat-input/utils/editor-state-to-plain-text'
|
||||||
import { useApp } from '../contexts/AppContext'
|
import { useApp } from '../contexts/AppContext'
|
||||||
|
import { useSettings } from '../contexts/SettingsContext'
|
||||||
import { ChatManager } from '../database/json/chat/ChatManager'
|
import { ChatManager } from '../database/json/chat/ChatManager'
|
||||||
import { deserializeChatMessage, serializeChatMessage } from '../database/json/utils'
|
import { deserializeChatMessage, serializeChatMessage } from '../database/json/utils'
|
||||||
|
import { WorkspaceManager } from '../database/json/workspace/WorkspaceManager'
|
||||||
import { ChatConversationMeta, ChatMessage, ChatUserMessage } from '../types/chat'
|
import { ChatConversationMeta, ChatMessage, ChatUserMessage } from '../types/chat'
|
||||||
|
|
||||||
type UseChatHistory = {
|
type UseChatHistory = {
|
||||||
@ -22,7 +24,9 @@ type UseChatHistory = {
|
|||||||
|
|
||||||
export function useChatHistory(): UseChatHistory {
|
export function useChatHistory(): UseChatHistory {
|
||||||
const app = useApp()
|
const app = useApp()
|
||||||
const chatManager = useMemo(() => new ChatManager(app), [app])
|
const { settings } = useSettings()
|
||||||
|
const workspaceManager = useMemo(() => new WorkspaceManager(app), [app])
|
||||||
|
const chatManager = useMemo(() => new ChatManager(app, workspaceManager), [app, workspaceManager])
|
||||||
|
|
||||||
const [chatList, setChatList] = useState<ChatConversationMeta[]>([])
|
const [chatList, setChatList] = useState<ChatConversationMeta[]>([])
|
||||||
|
|
||||||
@ -31,6 +35,9 @@ export function useChatHistory(): UseChatHistory {
|
|||||||
setChatList(conversations)
|
setChatList(conversations)
|
||||||
}, [chatManager])
|
}, [chatManager])
|
||||||
|
|
||||||
|
// 获取当前工作区
|
||||||
|
const currentWorkspace = settings.workspace || 'vault'
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
void fetchChatList()
|
void fetchChatList()
|
||||||
}, [fetchChatList])
|
}, [fetchChatList])
|
||||||
@ -52,7 +59,7 @@ export function useChatHistory(): UseChatHistory {
|
|||||||
} else {
|
} else {
|
||||||
const firstUserMessage = messages.find((v) => v.role === 'user') as ChatUserMessage
|
const firstUserMessage = messages.find((v) => v.role === 'user') as ChatUserMessage
|
||||||
|
|
||||||
const newChat = await chatManager.createChat({
|
await chatManager.createChat({
|
||||||
id,
|
id,
|
||||||
title: firstUserMessage?.content
|
title: firstUserMessage?.content
|
||||||
? editorStateToPlainText(firstUserMessage.content).substring(
|
? editorStateToPlainText(firstUserMessage.content).substring(
|
||||||
@ -61,6 +68,7 @@ export function useChatHistory(): UseChatHistory {
|
|||||||
)
|
)
|
||||||
: 'New chat',
|
: 'New chat',
|
||||||
messages: serializedMessages,
|
messages: serializedMessages,
|
||||||
|
workspace: currentWorkspace,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,7 +79,7 @@ export function useChatHistory(): UseChatHistory {
|
|||||||
maxWait: 1000,
|
maxWait: 1000,
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
[chatManager, fetchChatList],
|
[chatManager, fetchChatList, settings, workspaceManager],
|
||||||
)
|
)
|
||||||
|
|
||||||
const deleteConversation = useCallback(
|
const deleteConversation = useCallback(
|
||||||
|
|||||||
43
src/main.ts
43
src/main.ts
@ -1,7 +1,7 @@
|
|||||||
// @ts-nocheck
|
// @ts-nocheck
|
||||||
import { EditorView } from '@codemirror/view'
|
import { EditorView } from '@codemirror/view'
|
||||||
// import { PGlite } from '@electric-sql/pglite'
|
// import { PGlite } from '@electric-sql/pglite'
|
||||||
import { App, Editor, MarkdownView, Modal, Notice, Plugin, TFile } from 'obsidian'
|
import { Editor, MarkdownView, Modal, Notice, Plugin, TFile } from 'obsidian'
|
||||||
|
|
||||||
import { ApplyView } from './ApplyView'
|
import { ApplyView } from './ApplyView'
|
||||||
import { ChatView } from './ChatView'
|
import { ChatView } from './ChatView'
|
||||||
@ -419,6 +419,47 @@ export default class InfioPlugin extends Plugin {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 添加本地嵌入测试命令
|
||||||
|
this.addCommand({
|
||||||
|
id: 'test-local-embed',
|
||||||
|
name: '测试本地嵌入模型',
|
||||||
|
callback: async () => {
|
||||||
|
try {
|
||||||
|
// 动态导入嵌入管理器
|
||||||
|
const { embeddingManager } = await import('./embedworker/index');
|
||||||
|
|
||||||
|
// 加载模型
|
||||||
|
await embeddingManager.loadModel("Xenova/all-MiniLM-L6-v2", true);
|
||||||
|
|
||||||
|
// 测试嵌入 "hello world"
|
||||||
|
const testText = "hello world";
|
||||||
|
|
||||||
|
const result = await embeddingManager.embed(testText);
|
||||||
|
|
||||||
|
// 显示结果
|
||||||
|
const resultMessage = `
|
||||||
|
嵌入测试完成!
|
||||||
|
文本: "${testText}"
|
||||||
|
Token 数量: ${result.tokens}
|
||||||
|
向量维度: ${result.vec.length}
|
||||||
|
向量前4个值: [${result.vec.slice(0, 4).map(v => v.toFixed(4)).join(', ')}...]
|
||||||
|
`.trim();
|
||||||
|
|
||||||
|
console.log('本地嵌入测试结果:', result);
|
||||||
|
|
||||||
|
// 创建模态框显示结果
|
||||||
|
const modal = new Modal(this.app);
|
||||||
|
modal.titleEl.setText('本地嵌入测试结果');
|
||||||
|
modal.contentEl.createEl('pre', { text: resultMessage });
|
||||||
|
modal.open();
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('嵌入测试失败:', error);
|
||||||
|
new Notice(`嵌入测试失败: ${error.message}`, 5000);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
onunload() {
|
onunload() {
|
||||||
|
|||||||
@ -81,4 +81,5 @@ export type ChatConversationMeta = {
|
|||||||
title: string
|
title: string
|
||||||
createdAt: number
|
createdAt: number
|
||||||
updatedAt: number
|
updatedAt: number
|
||||||
|
workspace?: string
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
// @ts-expect-error - parse5 and JSON5 types are not perfectly aligned with the dynamic parsing logic
|
// @ts-nocheck
|
||||||
import JSON5 from 'json5'
|
import JSON5 from 'json5'
|
||||||
import { parseFragment } from 'parse5'
|
import { parseFragment } from 'parse5'
|
||||||
|
|
||||||
|
|||||||
@ -71,5 +71,5 @@
|
|||||||
// }
|
// }
|
||||||
},
|
},
|
||||||
// Specify which files to include in compilation
|
// Specify which files to include in compilation
|
||||||
"include": ["src/**/*.ts", "src/**/*.tsx", "__mocks__"]
|
"include": ["src/**/*.ts", "src/**/*.tsx", "__mocks__", "src/embedworker/*.worker.ts", "src/embedworker/*.worker.js"]
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user