add local embed

This commit is contained in:
duanfuxiang 2025-07-04 09:28:12 +08:00
parent cd65d6b3de
commit 65c5df3d22
22 changed files with 2156 additions and 195 deletions

View File

@ -61,6 +61,7 @@
"@electric-sql/pglite": "0.2.14", "@electric-sql/pglite": "0.2.14",
"@google/genai": "^1.2.0", "@google/genai": "^1.2.0",
"@google/generative-ai": "^0.21.0", "@google/generative-ai": "^0.21.0",
"@huggingface/transformers": "^3.6.1",
"@langchain/core": "^0.3.26", "@langchain/core": "^0.3.26",
"@lexical/clipboard": "^0.17.1", "@lexical/clipboard": "^0.17.1",
"@lexical/react": "^0.17.1", "@lexical/react": "^0.17.1",
@ -74,6 +75,7 @@
"@radix-ui/react-tooltip": "^1.1.3", "@radix-ui/react-tooltip": "^1.1.3",
"@tanstack/react-query": "^5.56.2", "@tanstack/react-query": "^5.56.2",
"@types/mermaid": "^9.2.0", "@types/mermaid": "^9.2.0",
"@xenova/transformers": "^2.17.2",
"axios": "^1.8.3", "axios": "^1.8.3",
"chokidar": "^4.0.3", "chokidar": "^4.0.3",
"clsx": "^2.1.1", "clsx": "^2.1.1",
@ -119,6 +121,7 @@
"remark-gfm": "^4.0.0", "remark-gfm": "^4.0.0",
"shell-env": "^4.0.1", "shell-env": "^4.0.1",
"simple-git": "^3.27.0", "simple-git": "^3.27.0",
"smart-embed-model": "^1.0.7",
"string-similarity": "^4.0.4", "string-similarity": "^4.0.4",
"styled-components": "^6.1.19", "styled-components": "^6.1.19",
"uuid": "^10.0.0", "uuid": "^10.0.0",

947
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,8 @@
import { CheckSquare, Clock, CopyPlus, MessageSquare, Pencil, Search, Sparkles, Square, Trash2 } from 'lucide-react' import { CheckSquare, Clock, CopyPlus, Globe, MessageSquare, Pencil, Search, Sparkles, Square, Trash2 } from 'lucide-react'
import { Notice } from 'obsidian' import { Notice } from 'obsidian'
import React, { useMemo, useRef, useState } from 'react' import React, { useMemo, useRef, useState } from 'react'
import { useSettings } from '../../contexts/SettingsContext'
import { useChatHistory } from '../../hooks/use-chat-history' import { useChatHistory } from '../../hooks/use-chat-history'
import { t } from '../../lang/helpers' import { t } from '../../lang/helpers'
import { ChatConversationMeta } from '../../types/chat' import { ChatConversationMeta } from '../../types/chat'
@ -29,6 +30,15 @@ const ChatHistoryView = ({
// search term // search term
const [searchTerm, setSearchTerm] = useState('') const [searchTerm, setSearchTerm] = useState('')
const { settings } = useSettings()
const currentWorkspace = React.useMemo(() => {
return settings.workspace || 'vault'
}, [settings.workspace])
// workspace filter state
const [filterByWorkspace, setFilterByWorkspace] = useState(false)
// editing conversation id // editing conversation id
const [editingConversationId, setEditingConversationId] = useState<string | null>(null) const [editingConversationId, setEditingConversationId] = useState<string | null>(null)
@ -62,16 +72,33 @@ const ChatHistoryView = ({
setSearchTerm(e.target.value) setSearchTerm(e.target.value)
} }
// toggle workspace filter
const toggleWorkspaceFilter = () => {
setFilterByWorkspace(!filterByWorkspace)
}
// filter conversations list // filter conversations list
const filteredConversations = useMemo(() => { const filteredConversations = useMemo(() => {
if (!searchTerm.trim()) { console.log('filteredConversations', chatList)
return chatList let filtered = chatList
// Apply search filter
if (searchTerm.trim()) {
filtered = filtered.filter(
conversation =>
conversation.title.toLowerCase().includes(searchTerm.toLowerCase())
)
} }
return chatList.filter(
conversation => // Apply workspace filter
conversation.title.toLowerCase().includes(searchTerm.toLowerCase()) if (filterByWorkspace) {
) filtered = filtered.filter(
}, [chatList, searchTerm]) conversation => conversation.workspace === currentWorkspace
)
}
return filtered
}, [chatList, searchTerm, filterByWorkspace, currentWorkspace])
// toggle selection mode // toggle selection mode
const toggleSelectionMode = () => { const toggleSelectionMode = () => {
@ -285,6 +312,18 @@ const ChatHistoryView = ({
/> />
</div> </div>
{/* workspace filter */}
<div className="infio-chat-history-workspace-filter">
<button
onClick={toggleWorkspaceFilter}
className={`infio-chat-history-workspace-filter-btn ${filterByWorkspace ? 'active' : ''}`}
title={filterByWorkspace ? '显示所有对话' : '只显示当前工作区对话'}
>
<Globe size={14} />
</button>
</div>
{/* conversations list */} {/* conversations list */}
<div className="infio-chat-history-list"> <div className="infio-chat-history-list">
{filteredConversations.length === 0 ? ( {filteredConversations.length === 0 ? (
@ -353,6 +392,11 @@ const ChatHistoryView = ({
{formatDate(conversation.updatedAt)} {formatDate(conversation.updatedAt)}
</div> </div>
<div className="infio-chat-history-conversation-title">{conversation.title}</div> <div className="infio-chat-history-conversation-title">{conversation.title}</div>
{conversation.workspace && (
<div className="infio-chat-history-workspace">
: {conversation.workspace}
</div>
)}
</div> </div>
{!selectionMode && ( {!selectionMode && (
<div className="infio-chat-history-actions"> <div className="infio-chat-history-actions">
@ -426,6 +470,7 @@ const ChatHistoryView = ({
flex-shrink: 0; flex-shrink: 0;
} }
.infio-chat-history-filter-btn,
.infio-chat-history-cleanup-btn, .infio-chat-history-cleanup-btn,
.infio-chat-history-selection-btn { .infio-chat-history-selection-btn {
display: flex !important; display: flex !important;
@ -444,12 +489,14 @@ const ChatHistoryView = ({
box-sizing: border-box; box-sizing: border-box;
} }
.infio-chat-history-filter-btn:hover,
.infio-chat-history-cleanup-btn:hover, .infio-chat-history-cleanup-btn:hover,
.infio-chat-history-selection-btn:hover { .infio-chat-history-selection-btn:hover {
background-color: var(--background-modifier-hover, #f5f5f5); background-color: var(--background-modifier-hover, #f5f5f5);
border-color: var(--background-modifier-border-hover, #d0d0d0); border-color: var(--background-modifier-border-hover, #d0d0d0);
} }
.infio-chat-history-filter-btn.active,
.infio-chat-history-selection-btn.active { .infio-chat-history-selection-btn.active {
background-color: var(--interactive-accent, #007acc); background-color: var(--interactive-accent, #007acc);
color: var(--text-on-accent, #ffffff); color: var(--text-on-accent, #ffffff);
@ -539,7 +586,6 @@ const ChatHistoryView = ({
border: 1px solid var(--background-modifier-border); border: 1px solid var(--background-modifier-border);
border-radius: var(--radius-s); border-radius: var(--radius-s);
padding: 6px 12px; padding: 6px 12px;
margin-bottom: var(--size-4-3);
transition: all 0.2s ease; transition: all 0.2s ease;
height: 36px; height: 36px;
max-width: 100%; max-width: 100%;
@ -675,6 +721,13 @@ const ChatHistoryView = ({
font-size: 12px; font-size: 12px;
} }
.infio-chat-history-workspace {
color: var(--text-muted);
font-size: 11px;
margin-top: 2px;
opacity: 0.8;
}
.infio-chat-history-actions { .infio-chat-history-actions {
display: flex; display: flex;
gap: 4px; gap: 4px;
@ -768,6 +821,38 @@ const ChatHistoryView = ({
.infio-chat-history-cancel-btn:hover { .infio-chat-history-cancel-btn:hover {
background-color: var(--background-modifier-hover); background-color: var(--background-modifier-hover);
} }
.infio-chat-history-workspace-filter {
display: flex;
justify-content: flex-start;
align-items: center;
margin-bottom: 12px;
}
.infio-chat-history-workspace-filter-btn {
display: flex;
align-items: center;
gap: 6px;
background-color: transparent;
border: 1px solid var(--background-modifier-border);
color: var(--text-muted);
padding: 6px 12px;
border-radius: var(--radius-s);
cursor: pointer;
font-size: var(--font-ui-small);
transition: all 0.2s ease;
}
.infio-chat-history-workspace-filter-btn:hover {
background-color: var(--background-modifier-hover);
color: var(--text-normal);
}
.infio-chat-history-workspace-filter-btn.active {
background-color: var(--interactive-accent);
color: var(--text-on-accent);
border-color: var(--interactive-accent);
}
`} `}
</style> </style>
</div> </div>

View File

@ -879,7 +879,7 @@ const Chat = forwardRef<ChatRef, ChatProps>((props, ref) => {
// Execute the transformation using the TransEngine // Execute the transformation using the TransEngine
const transformationResult = await transEngine.runTransformation({ const transformationResult = await transEngine.runTransformation({
filePath: toolArgs.path, filePath: toolArgs.path,
transformationType: transformationType, transformationType: transformationType as TransformationType,
model: { model: {
provider: settings.applyModelProvider, provider: settings.applyModelProvider,
modelId: settings.applyModelId, modelId: settings.applyModelId,

View File

@ -148,7 +148,7 @@ const SearchView = () => {
minSimilarity: 0.3, minSimilarity: 0.3,
}) })
setInsightResults(results) setInsightResults(results as any)
setSearchResults([]) setSearchResults([])
} }
} catch (error) { } catch (error) {

View File

@ -59,8 +59,9 @@ const WorkspaceEditModal = ({
setAvailableFolders(folders.sort()) setAvailableFolders(folders.sort())
// 直接使用 Obsidian 的内置接口获取所有标签 // 直接使用 Obsidian 的内置接口获取所有标签
try { try {
const tagsObject = app.metadataCache.getTags() // 获取所有标签 {'#tag1': 2, '#tag2': 4} // @ts-ignore
const tagsObject = app.metadataCache.getTags() as Record<string, number> // 获取所有标签 {'#tag1': 2, '#tag2': 4}
const tags = Object.keys(tagsObject).sort() const tags = Object.keys(tagsObject).sort()
setAvailableTags(tags) setAvailableTags(tags)
} catch (error) { } catch (error) {

View File

@ -112,5 +112,5 @@ export function getRulesSection(
if (mode === 'learn') { if (mode === 'learn') {
return getLearnModeRulesSection(cwd, searchTool); return getLearnModeRulesSection(cwd, searchTool);
} }
return getObsidianRulesSection(mode, cwd, searchTool, supportsComputerUse, diffStrategy, experiments); return getObsidianRulesSection(mode, cwd, searchTool);
} }

View File

@ -21,8 +21,5 @@ By waiting for and carefully considering the user's response after each tool use
} }
export function getToolUseGuidelinesSection(mode?: string): string { export function getToolUseGuidelinesSection(mode?: string): string {
if (mode === 'learn') {
return getLearnModeToolUseGuidelines();
}
return getDefaultToolUseGuidelines(); return getDefaultToolUseGuidelines();
} }

View File

@ -61,7 +61,7 @@ export type ToolName = keyof typeof TOOL_DISPLAY_NAMES
// Tool helper functions // Tool helper functions
export function getToolName(toolConfig: string | readonly [ToolName, ...unknown[]]): ToolName { export function getToolName(toolConfig: string | readonly [ToolName, ...unknown[]]): ToolName {
return typeof toolConfig === "string" ? toolConfig : toolConfig[0] return typeof toolConfig === "string" ? toolConfig as ToolName : toolConfig[0]
} }
export function getToolOptions(toolConfig: string | readonly [ToolName, ...unknown[]]): unknown { export function getToolOptions(toolConfig: string | readonly [ToolName, ...unknown[]]): unknown {

View File

@ -981,8 +981,9 @@ export class TransEngine {
} }
break; break;
} }
case TransformationType.DENSE_SUMMARY:
case TransformationType.CONCISE_DENSE_SUMMARY: // 新的摘要类型不需要特殊的后处理,保持原样
break;
case TransformationType.HIERARCHICAL_SUMMARY: case TransformationType.HIERARCHICAL_SUMMARY:
// 新的摘要类型不需要特殊的后处理,保持原样 // 新的摘要类型不需要特殊的后处理,保持原样
break; break;
@ -1237,7 +1238,7 @@ export class TransEngine {
const cacheResult = await this.checkDatabaseCache( const cacheResult = await this.checkDatabaseCache(
fileMetadata.sourcePath, fileMetadata.sourcePath,
fileMetadata.sourceMtime, fileMetadata.sourceMtime,
TransformationType.CONCISE_DENSE_SUMMARY TransformationType.DENSE_SUMMARY
) )
if (cacheResult.foundCache && cacheResult.result.success && cacheResult.result.result) { if (cacheResult.foundCache && cacheResult.result.success && cacheResult.result.result) {
@ -1273,7 +1274,7 @@ export class TransEngine {
// 保存到数据库 // 保存到数据库
await this.saveResultToDatabase( await this.saveResultToDatabase(
summary, summary,
TransformationType.CONCISE_DENSE_SUMMARY, TransformationType.DENSE_SUMMARY,
fileMetadata.sourcePath, fileMetadata.sourcePath,
fileMetadata.sourceMtime, fileMetadata.sourceMtime,
'document' 'document'
@ -1295,7 +1296,7 @@ export class TransEngine {
const messages: RequestMessage[] = [ const messages: RequestMessage[] = [
{ {
role: 'system', role: 'system',
content: CONCISE_DENSE_SUMMARY_PROMPT content: DENSE_SUMMARY_PROMPT
}, },
{ {
role: 'user', role: 'user',
@ -1308,7 +1309,7 @@ export class TransEngine {
throw new Error(`生成摘要失败: ${result.error.message}`) throw new Error(`生成摘要失败: ${result.error.message}`)
} }
return this.postProcessResult(result.value, TransformationType.CONCISE_DENSE_SUMMARY) return this.postProcessResult(result.value, TransformationType.DENSE_SUMMARY)
} }
/** /**

View File

@ -5,6 +5,7 @@ import { ChatConversationMeta } from '../../../types/chat'
import { AbstractJsonRepository } from '../base' import { AbstractJsonRepository } from '../base'
import { CHAT_DIR, ROOT_DIR } from '../constants' import { CHAT_DIR, ROOT_DIR } from '../constants'
import { EmptyChatTitleException } from '../exception' import { EmptyChatTitleException } from '../exception'
import { WorkspaceManager } from '../workspace/WorkspaceManager'
import { import {
CHAT_SCHEMA_VERSION, CHAT_SCHEMA_VERSION,
@ -16,27 +17,34 @@ export class ChatManager extends AbstractJsonRepository<
ChatConversation, ChatConversation,
ChatConversationMeta ChatConversationMeta
> { > {
constructor(app: App) { private workspaceManager?: WorkspaceManager
constructor(app: App, workspaceManager?: WorkspaceManager) {
super(app, `${ROOT_DIR}/${CHAT_DIR}`) super(app, `${ROOT_DIR}/${CHAT_DIR}`)
this.workspaceManager = workspaceManager
} }
protected generateFileName(chat: ChatConversation): string { protected generateFileName(chat: ChatConversation): string {
// Format: v{schemaVersion}_{title}_{updatedAt}_{id}.json // 新格式: v{schemaVersion}_{title}_{updatedAt}_{id}_{workspaceId}.json
// 如果没有工作区,使用 'vault' 作为默认值
const encodedTitle = encodeURIComponent(chat.title) const encodedTitle = encodeURIComponent(chat.title)
return `v${chat.schemaVersion}_${encodedTitle}_${chat.updatedAt}_${chat.id}.json` const workspaceId = chat.workspace || 'vault'
return `v${chat.schemaVersion}_${encodedTitle}_${chat.updatedAt}_${chat.id}_${workspaceId}.json`
} }
protected parseFileName(fileName: string): ChatConversationMeta | null { protected parseFileName(fileName: string): ChatConversationMeta | null {
// Parse: v{schemaVersion}_{title}_{updatedAt}_{id}.json // 使用一个正则表达式,工作区部分为可选: v{schemaVersion}_{title}_{updatedAt}_{id}_{workspaceId}?.json
const regex = new RegExp( const regex = new RegExp(
`^v${CHAT_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)\\.json$`, `^v${CHAT_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)(?:_([^_]+))?\\.json$`,
) )
const match = fileName.match(regex) const match = fileName.match(regex)
if (!match) return null if (!match) return null
const title = decodeURIComponent(match[1]) const title = decodeURIComponent(match[1])
const updatedAt = parseInt(match[2], 10) const updatedAt = parseInt(match[2], 10)
const id = match[3] const id = match[3]
const workspaceId = match[4] // 可能为undefined老格式
return { return {
id, id,
@ -44,6 +52,8 @@ export class ChatManager extends AbstractJsonRepository<
title, title,
updatedAt, updatedAt,
createdAt: 0, createdAt: 0,
// 如果没有工作区信息老格式则认为是vault全局消息
workspace: workspaceId === 'vault' ? undefined : workspaceId,
} }
} }
@ -66,6 +76,20 @@ export class ChatManager extends AbstractJsonRepository<
} }
await this.create(newChat) await this.create(newChat)
// 如果有工作区信息,添加到工作区的聊天历史中
if (newChat.workspace && this.workspaceManager) {
try {
await this.workspaceManager.addChatToWorkspace(
newChat.workspace,
newChat.id,
newChat.title
)
} catch (error) {
console.error('Failed to add chat to workspace:', error)
}
}
return newChat return newChat
} }
@ -102,6 +126,23 @@ export class ChatManager extends AbstractJsonRepository<
} }
await this.update(chat, updatedChat) await this.update(chat, updatedChat)
// 如果标题或工作区发生变化,更新工作区的聊天历史
if (this.workspaceManager && (updates.title !== undefined || updates.workspace !== undefined)) {
const workspaceId = updatedChat.workspace || chat.workspace
if (workspaceId) {
try {
await this.workspaceManager.addChatToWorkspace(
workspaceId,
updatedChat.id,
updatedChat.title
)
} catch (error) {
console.error('Failed to update chat in workspace:', error)
}
}
}
return updatedChat return updatedChat
} }
@ -111,9 +152,22 @@ export class ChatManager extends AbstractJsonRepository<
if (targetsToDelete.length === 0) return false if (targetsToDelete.length === 0) return false
// 获取聊天的工作区信息(从第一个匹配的元数据中获取)
const chatToDelete = await this.findById(id)
const workspaceId = chatToDelete?.workspace
// Delete all files associated with this ID // Delete all files associated with this ID
await Promise.all(targetsToDelete.map(meta => this.delete(meta.fileName))) await Promise.all(targetsToDelete.map(meta => this.delete(meta.fileName)))
// 从工作区的聊天历史中移除
if (workspaceId && this.workspaceManager) {
try {
await this.workspaceManager.removeChatFromWorkspace(workspaceId, id)
} catch (error) {
console.error('Failed to remove chat from workspace:', error)
}
}
return true return true
} }
@ -126,7 +180,10 @@ export class ChatManager extends AbstractJsonRepository<
if (!chatsById.has(meta.id)) { if (!chatsById.has(meta.id)) {
chatsById.set(meta.id, []) chatsById.set(meta.id, [])
} }
chatsById.get(meta.id)!.push(meta) const chatGroup = chatsById.get(meta.id)
if (chatGroup) {
chatGroup.push(meta)
}
} }
const filesToDelete: string[] = [] const filesToDelete: string[] = []
@ -151,11 +208,12 @@ export class ChatManager extends AbstractJsonRepository<
return filesToDelete.length return filesToDelete.length
} }
public async listChats(): Promise<ChatConversationMeta[]> { public async listChats(workspaceFilter?: string): Promise<ChatConversationMeta[]> {
console.log('listChats', workspaceFilter)
const metadata = await this.listMetadata() const metadata = await this.listMetadata()
// Use a Map to store the latest version of each chat by ID. // Use a Map to store the latest version of each chat by ID.
const latestChats = new Map<string, ChatConversationMeta>() const latestChats = new Map<string, ChatConversationMeta & { fileName: string }>()
for (const meta of metadata) { for (const meta of metadata) {
const existing = latestChats.get(meta.id) const existing = latestChats.get(meta.id)
@ -165,7 +223,27 @@ export class ChatManager extends AbstractJsonRepository<
} }
const uniqueMetadata = Array.from(latestChats.values()) const uniqueMetadata = Array.from(latestChats.values())
const sorted = uniqueMetadata.sort((a, b) => b.updatedAt - a.updatedAt)
// 将metadata转换为ChatConversationMeta格式
const chatMetadata: ChatConversationMeta[] = uniqueMetadata.map((meta) => ({
id: meta.id,
schemaVersion: meta.schemaVersion,
title: meta.title,
updatedAt: meta.updatedAt,
createdAt: meta.createdAt,
workspace: meta.workspace
}))
// 如果指定了工作区过滤器,则过滤对话
let filteredMetadata = chatMetadata
if (workspaceFilter !== undefined && workspaceFilter !== 'vault') {
// 获取指定工作区的对话
filteredMetadata = chatMetadata.filter(meta =>
meta.workspace === workspaceFilter
)
}
const sorted = filteredMetadata.sort((a, b) => b.updatedAt - a.updatedAt)
return sorted return sorted
} }
} }

View File

@ -9,6 +9,7 @@ export type ChatConversation = {
createdAt: number createdAt: number
updatedAt: number updatedAt: number
schemaVersion: number schemaVersion: number
workspace?: string // 工作区ID可选字段用于向后兼容
} }
export type ChatConversationMetadata = { export type ChatConversationMetadata = {
@ -16,4 +17,5 @@ export type ChatConversationMetadata = {
title: string title: string
updatedAt: number updatedAt: number
schemaVersion: number schemaVersion: number
workspace?: string // 工作区ID可选字段用于向后兼容
} }

View File

@ -5,181 +5,181 @@ import { AbstractJsonRepository } from '../base'
import { ROOT_DIR, WORKSPACE_DIR } from '../constants' import { ROOT_DIR, WORKSPACE_DIR } from '../constants'
import { import {
WORKSPACE_SCHEMA_VERSION, WORKSPACE_SCHEMA_VERSION,
Workspace, Workspace,
WorkspaceMetadata WorkspaceMetadata
} from './types' } from './types'
export class WorkspaceManager extends AbstractJsonRepository< export class WorkspaceManager extends AbstractJsonRepository<
Workspace, Workspace,
WorkspaceMetadata WorkspaceMetadata
> { > {
constructor(app: App) { constructor(app: App) {
super(app, `${ROOT_DIR}/${WORKSPACE_DIR}`) super(app, `${ROOT_DIR}/${WORKSPACE_DIR}`)
} }
protected generateFileName(workspace: Workspace): string { protected generateFileName(workspace: Workspace): string {
// Format: v{schemaVersion}_{name}_{updatedAt}_{id}.json // Format: v{schemaVersion}_{name}_{updatedAt}_{id}.json
const encodedName = encodeURIComponent(workspace.name) const encodedName = encodeURIComponent(workspace.name)
return `v${workspace.schemaVersion}_${encodedName}_${workspace.updatedAt}_${workspace.id}.json` return `v${workspace.schemaVersion}_${encodedName}_${workspace.updatedAt}_${workspace.id}.json`
} }
protected parseFileName(fileName: string): WorkspaceMetadata | null { protected parseFileName(fileName: string): WorkspaceMetadata | null {
// Parse: v{schemaVersion}_{name}_{updatedAt}_{id}.json // Parse: v{schemaVersion}_{name}_{updatedAt}_{id}.json
const regex = new RegExp( const regex = new RegExp(
`^v${WORKSPACE_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)\\.json$`, `^v${WORKSPACE_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)\\.json$`,
) )
const match = fileName.match(regex) const match = fileName.match(regex)
if (!match) return null if (!match) return null
const name = decodeURIComponent(match[1]) const name = decodeURIComponent(match[1])
const updatedAt = parseInt(match[2], 10) const updatedAt = parseInt(match[2], 10)
const id = match[3] const id = match[3]
return { return {
id, id,
name, name,
updatedAt, updatedAt,
createdAt: 0, createdAt: 0,
schemaVersion: WORKSPACE_SCHEMA_VERSION, schemaVersion: WORKSPACE_SCHEMA_VERSION,
} }
} }
public async createWorkspace( public async createWorkspace(
initialData: Partial<Workspace>, initialData: Partial<Workspace>,
): Promise<Workspace> { ): Promise<Workspace> {
const now = Date.now() const now = Date.now()
const newWorkspace: Workspace = { const newWorkspace: Workspace = {
id: uuidv4(), id: uuidv4(),
name: 'New Workspace', name: 'New Workspace',
content: [], content: [],
chatHistory: [], chatHistory: [],
metadata: {}, metadata: {},
createdAt: now, createdAt: now,
updatedAt: now, updatedAt: now,
schemaVersion: WORKSPACE_SCHEMA_VERSION, schemaVersion: WORKSPACE_SCHEMA_VERSION,
...initialData, ...initialData,
} }
await this.create(newWorkspace) await this.create(newWorkspace)
return newWorkspace return newWorkspace
} }
public async findById(id: string): Promise<Workspace | null> { public async findById(id: string): Promise<Workspace | null> {
const allMetadata = await this.listMetadata() const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.id === id) const targetMetadata = allMetadata.find((meta) => meta.id === id)
if (!targetMetadata) return null if (!targetMetadata) return null
return this.read(targetMetadata.fileName) return this.read(targetMetadata.fileName)
} }
public async findByName(name: string): Promise<Workspace | null> { public async findByName(name: string): Promise<Workspace | null> {
const allMetadata = await this.listMetadata() const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.name === name) const targetMetadata = allMetadata.find((meta) => meta.name === name)
if (!targetMetadata) return null if (!targetMetadata) return null
return this.read(targetMetadata.fileName) return this.read(targetMetadata.fileName)
} }
public async updateWorkspace( public async updateWorkspace(
id: string, id: string,
updates: Partial< updates: Partial<
Omit<Workspace, 'id' | 'createdAt' | 'updatedAt' | 'schemaVersion'> Omit<Workspace, 'id' | 'createdAt' | 'updatedAt' | 'schemaVersion'>
>, >,
): Promise<Workspace | null> { ): Promise<Workspace | null> {
const workspace = await this.findById(id) const workspace = await this.findById(id)
if (!workspace) return null if (!workspace) return null
const updatedWorkspace: Workspace = { const updatedWorkspace: Workspace = {
...workspace, ...workspace,
...updates, ...updates,
updatedAt: Date.now(), updatedAt: Date.now(),
} }
await this.update(workspace, updatedWorkspace) await this.update(workspace, updatedWorkspace)
return updatedWorkspace return updatedWorkspace
} }
public async deleteWorkspace(id: string): Promise<boolean> { public async deleteWorkspace(id: string): Promise<boolean> {
const allMetadata = await this.listMetadata() const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.id === id) const targetMetadata = allMetadata.find((meta) => meta.id === id)
if (!targetMetadata) return false if (!targetMetadata) return false
await this.delete(targetMetadata.fileName) await this.delete(targetMetadata.fileName)
return true return true
} }
public async listWorkspaces(): Promise<WorkspaceMetadata[]> { public async listWorkspaces(): Promise<WorkspaceMetadata[]> {
const metadata = await this.listMetadata() const metadata = await this.listMetadata()
const sorted = metadata.sort((a, b) => b.updatedAt - a.updatedAt) const sorted = metadata.sort((a, b) => b.updatedAt - a.updatedAt)
return sorted return sorted
} }
public async addChatToWorkspace( public async addChatToWorkspace(
workspaceId: string, workspaceName: string,
chatId: string, chatId: string,
chatTitle: string chatTitle: string
): Promise<Workspace | null> { ): Promise<Workspace | null> {
const workspace = await this.findById(workspaceId) const workspace = await this.findByName(workspaceName)
if (!workspace) return null if (!workspace) return null
const existingChatIndex = workspace.chatHistory.findIndex( const existingChatIndex = workspace.chatHistory.findIndex(
chat => chat.id === chatId chat => chat.id === chatId
) )
if (existingChatIndex >= 0) { if (existingChatIndex >= 0) {
// 更新已存在的聊天标题 // 更新已存在的聊天标题
workspace.chatHistory[existingChatIndex].title = chatTitle workspace.chatHistory[existingChatIndex].title = chatTitle
} else { } else {
// 添加新聊天 // 添加新聊天
workspace.chatHistory.push({ id: chatId, title: chatTitle }) workspace.chatHistory.push({ id: chatId, title: chatTitle })
} }
return this.updateWorkspace(workspaceId, { return this.updateWorkspace(workspace.id, {
chatHistory: workspace.chatHistory chatHistory: workspace.chatHistory
}) })
} }
public async removeChatFromWorkspace( public async removeChatFromWorkspace(
workspaceId: string, workspaceId: string,
chatId: string chatId: string
): Promise<Workspace | null> { ): Promise<Workspace | null> {
const workspace = await this.findById(workspaceId) const workspace = await this.findById(workspaceId)
if (!workspace) return null if (!workspace) return null
workspace.chatHistory = workspace.chatHistory.filter( workspace.chatHistory = workspace.chatHistory.filter(
chat => chat.id !== chatId chat => chat.id !== chatId
) )
return this.updateWorkspace(workspaceId, { return this.updateWorkspace(workspaceId, {
chatHistory: workspace.chatHistory chatHistory: workspace.chatHistory
}) })
} }
public async ensureDefaultVaultWorkspace(): Promise<Workspace> { public async ensureDefaultVaultWorkspace(): Promise<Workspace> {
// 检查是否已存在默认的 vault 工作区 // 检查是否已存在默认的 vault 工作区
const existingVault = await this.findByName('vault') const existingVault = await this.findByName('vault')
if (existingVault) { if (existingVault) {
return existingVault return existingVault
} }
// 创建默认的 vault 工作区 // 创建默认的 vault 工作区
const defaultWorkspace = await this.createWorkspace({ const defaultWorkspace = await this.createWorkspace({
name: 'vault', name: 'vault',
content: [ content: [
{ {
type: 'folder', type: 'folder',
content: '/' // 整个 vault 根目录 content: '/' // 整个 vault 根目录
} }
], ],
metadata: { metadata: {
isDefault: true, isDefault: true,
description: 'all vault as workspace' description: 'all vault as workspace'
} }
}) })
return defaultWorkspace return defaultWorkspace
} }
} }

View File

@ -0,0 +1,274 @@
// 导入完整的嵌入 Worker
// @ts-nocheck
import EmbedWorker from './embed.worker';
// 类型定义
export interface EmbedResult {
vec: number[];
tokens: number;
embed_input?: string;
}
export interface ModelLoadResult {
model_loaded: boolean;
}
export interface ModelUnloadResult {
model_unloaded: boolean;
}
export interface TokenCountResult {
tokens: number;
}
export class EmbeddingManager {
private worker: Worker;
private requests = new Map<number, { resolve: (value: any) => void; reject: (reason?: any) => void }>();
private nextRequestId = 0;
private isModelLoaded = false;
private currentModelId: string | null = null;
constructor() {
// 创建 Worker使用与 pgworker 相同的模式
this.worker = new EmbedWorker();
// 统一监听来自 Worker 的所有消息
this.worker.onmessage = (event) => {
const { id, result, error } = event.data;
// 根据返回的 id 找到对应的 Promise 回调
const request = this.requests.get(id);
if (request) {
if (error) {
request.reject(new Error(error));
} else {
request.resolve(result);
}
// 完成后从 Map 中删除
this.requests.delete(id);
}
};
this.worker.onerror = (error) => {
console.error("EmbeddingWorker error:", error);
// 拒绝所有待处理的请求
this.requests.forEach(request => {
request.reject(error);
});
this.requests.clear();
};
}
/**
* Worker Promise Promise
* @param method (e.g., 'load', 'embed_batch')
* @param params
*/
private postRequest<T>(method: string, params: any): Promise<T> {
return new Promise<T>((resolve, reject) => {
const id = this.nextRequestId++;
this.requests.set(id, { resolve, reject });
this.worker.postMessage({ method, params, id });
});
}
/**
* Worker
* @param modelId ID, 'TaylorAI/bge-micro-v2'
* @param useGpu 使GPU加速false
*/
public async loadModel(modelId: string, useGpu: boolean = false): Promise<ModelLoadResult> {
console.log(`Loading embedding model: ${modelId}, GPU: ${useGpu}`);
try {
// 如果已经加载了相同的模型,直接返回
if (this.isModelLoaded && this.currentModelId === modelId) {
console.log(`Model ${modelId} already loaded`);
return { model_loaded: true };
}
// 如果加载了不同的模型,先卸载
if (this.isModelLoaded && this.currentModelId !== modelId) {
console.log(`Unloading previous model: ${this.currentModelId}`);
await this.unloadModel();
}
const result = await this.postRequest<ModelLoadResult>('load', {
model_key: modelId,
use_gpu: useGpu
});
this.isModelLoaded = result.model_loaded;
this.currentModelId = result.model_loaded ? modelId : null;
if (result.model_loaded) {
console.log(`Model ${modelId} loaded successfully`);
}
return result;
} catch (error) {
console.error(`Failed to load model ${modelId}:`, error);
this.isModelLoaded = false;
this.currentModelId = null;
throw error;
}
}
/**
*
* @param texts
* @returns token
*/
public async embedBatch(texts: string[]): Promise<EmbedResult[]> {
if (!this.isModelLoaded) {
throw new Error('Model not loaded. Please call loadModel() first.');
}
if (!texts || texts.length === 0) {
return [];
}
console.log(`Generating embeddings for ${texts.length} texts`);
try {
const inputs = texts.map(text => ({ embed_input: text }));
const results = await this.postRequest<EmbedResult[]>('embed_batch', { inputs });
console.log(`Generated ${results.length} embeddings`);
return results;
} catch (error) {
console.error('Failed to generate embeddings:', error);
throw error;
}
}
/**
*
* @param text
* @returns token
*/
public async embed(text: string): Promise<EmbedResult> {
if (!text || text.trim().length === 0) {
throw new Error('Text cannot be empty');
}
const results = await this.embedBatch([text]);
if (results.length === 0) {
throw new Error('Failed to generate embedding');
}
return results[0];
}
/**
* token
* @param text
*/
public async countTokens(text: string): Promise<TokenCountResult> {
if (!this.isModelLoaded) {
throw new Error('Model not loaded. Please call loadModel() first.');
}
if (!text) {
return { tokens: 0 };
}
try {
return await this.postRequest<TokenCountResult>('count_tokens', text);
} catch (error) {
console.error('Failed to count tokens:', error);
throw error;
}
}
/**
*
*/
public async unloadModel(): Promise<ModelUnloadResult> {
if (!this.isModelLoaded) {
console.log('No model to unload');
return { model_unloaded: true };
}
try {
console.log(`Unloading model: ${this.currentModelId}`);
const result = await this.postRequest<ModelUnloadResult>('unload', {});
this.isModelLoaded = false;
this.currentModelId = null;
console.log('Model unloaded successfully');
return result;
} catch (error) {
console.error('Failed to unload model:', error);
// 即使卸载失败,也重置状态
this.isModelLoaded = false;
this.currentModelId = null;
throw error;
}
}
/**
*
*/
public get modelLoaded(): boolean {
return this.isModelLoaded;
}
/**
* ID
*/
public get currentModel(): string | null {
return this.currentModelId;
}
/**
*
*/
public getSupportedModels(): string[] {
return [
'Xenova/all-MiniLM-L6-v2',
'Xenova/bge-small-en-v1.5',
'Xenova/bge-base-en-v1.5',
'Xenova/jina-embeddings-v2-base-zh',
'Xenova/jina-embeddings-v2-small-en',
'Xenova/multilingual-e5-small',
'Xenova/multilingual-e5-base',
'Xenova/gte-small',
'Xenova/e5-small-v2',
'Xenova/e5-base-v2'
];
}
/**
*
*/
public getModelInfo(modelId: string): { dims: number; maxTokens: number; description: string } | null {
const modelInfoMap: Record<string, { dims: number; maxTokens: number; description: string }> = {
'Xenova/all-MiniLM-L6-v2': { dims: 384, maxTokens: 512, description: 'All-MiniLM-L6-v2 (推荐,轻量级)' },
'Xenova/bge-small-en-v1.5': { dims: 384, maxTokens: 512, description: 'BGE-small-en-v1.5' },
'Xenova/bge-base-en-v1.5': { dims: 768, maxTokens: 512, description: 'BGE-base-en-v1.5 (更高质量)' },
'Xenova/jina-embeddings-v2-base-zh': { dims: 768, maxTokens: 8192, description: 'Jina-v2-base-zh (中英双语)' },
'Xenova/jina-embeddings-v2-small-en': { dims: 512, maxTokens: 8192, description: 'Jina-v2-small-en' },
'Xenova/multilingual-e5-small': { dims: 384, maxTokens: 512, description: 'E5-small (多语言)' },
'Xenova/multilingual-e5-base': { dims: 768, maxTokens: 512, description: 'E5-base (多语言,更高质量)' },
'Xenova/gte-small': { dims: 384, maxTokens: 512, description: 'GTE-small' },
'Xenova/e5-small-v2': { dims: 384, maxTokens: 512, description: 'E5-small-v2' },
'Xenova/e5-base-v2': { dims: 768, maxTokens: 512, description: 'E5-base-v2 (更高质量)' }
};
return modelInfoMap[modelId] || null;
}
/**
* Worker
*/
public terminate() {
this.worker.terminate();
this.requests.clear();
this.isModelLoaded = false;
}
}

171
src/embedworker/README.md Normal file
View File

@ -0,0 +1,171 @@
# 本地嵌入功能
这个模块提供了在 Web Worker 中运行的本地嵌入功能,使用 Transformers.js 库来生成文本的向量表示。
## 功能特性
- 🚀 **高性能**: 在 Web Worker 中运行,不阻塞主线程
- 🔒 **隐私保护**: 完全本地运行,数据不离开设备
- 🎯 **多模型支持**: 支持多种预训练的嵌入模型
- 💾 **内存管理**: 自动管理模型加载和卸载
- 🔧 **类型安全**: 完整的 TypeScript 类型支持
## 快速开始
### 基本使用
```typescript
import { embeddingManager } from './embedworker';
// 加载模型
await embeddingManager.loadModel('Xenova/all-MiniLM-L6-v2');
// 生成单个文本的嵌入向量
const result = await embeddingManager.embed('Hello, world!');
console.log(result.vec); // [0.1234, -0.5678, ...]
console.log(result.tokens); // 3
// 批量生成嵌入向量
const texts = ['Hello', 'World', 'AI is amazing'];
const results = await embeddingManager.embedBatch(texts);
// 计算 token 数量
const tokenCount = await embeddingManager.countTokens('How many tokens?');
console.log(tokenCount.tokens); // 4
```
### 高级使用
```typescript
import { EmbeddingManager } from './embedworker';
// 创建自定义实例
const customEmbedding = new EmbeddingManager();
// 使用 GPU 加速(如果支持)
await customEmbedding.loadModel('Xenova/all-MiniLM-L6-v2', true);
// 检查模型状态
console.log(customEmbedding.modelLoaded); // true
console.log(customEmbedding.currentModel); // 'TaylorAI/bge-micro-v2'
// 获取支持的模型列表
const models = customEmbedding.getSupportedModels();
console.log(models);
// 获取模型信息
const modelInfo = customEmbedding.getModelInfo('Xenova/all-MiniLM-L6-v2');
console.log(modelInfo); // { dims: 384, maxTokens: 512, description: '...' }
// 切换模型
await customEmbedding.loadModel('Snowflake/snowflake-arctic-embed-xs');
// 清理资源
await customEmbedding.unloadModel();
customEmbedding.terminate();
```
## 支持的模型
| 模型 | 维度 | 最大Token | 描述 |
|------|------|-----------|------|
| Xenova/all-MiniLM-L6-v2 | 384 | 512 | All-MiniLM-L6-v2 (推荐,轻量级) |
| Xenova/bge-small-en-v1.5 | 384 | 512 | BGE-small-en-v1.5 |
| Xenova/bge-base-en-v1.5 | 768 | 512 | BGE-base-en-v1.5 (更高质量) |
| Xenova/jina-embeddings-v2-base-zh | 768 | 8192 | Jina-v2-base-zh (中英双语) |
| Xenova/jina-embeddings-v2-small-en | 512 | 8192 | Jina-v2-small-en |
| Xenova/multilingual-e5-small | 384 | 512 | E5-small (多语言) |
| Xenova/multilingual-e5-base | 768 | 512 | E5-base (多语言,更高质量) |
| Xenova/gte-small | 384 | 512 | GTE-small |
| Xenova/e5-small-v2 | 384 | 512 | E5-small-v2 |
| Xenova/e5-base-v2 | 768 | 512 | E5-base-v2 (更高质量) |
## API 参考
### EmbeddingManager
#### 方法
- `loadModel(modelId: string, useGpu?: boolean): Promise<ModelLoadResult>`
- 加载指定的嵌入模型
- `modelId`: 模型标识符
- `useGpu`: 是否使用 GPU 加速(默认 false
- `embed(text: string): Promise<EmbedResult>`
- 为单个文本生成嵌入向量
- 返回包含向量和 token 数量的结果
- `embedBatch(texts: string[]): Promise<EmbedResult[]>`
- 为多个文本批量生成嵌入向量
- 更高效的批处理方式
- `countTokens(text: string): Promise<TokenCountResult>`
- 计算文本的 token 数量
- `unloadModel(): Promise<ModelUnloadResult>`
- 卸载当前模型,释放内存
- `terminate(): void`
- 终止 Worker释放所有资源
#### 属性
- `modelLoaded: boolean` - 模型是否已加载
- `currentModel: string | null` - 当前加载的模型ID
#### 工具方法
- `getSupportedModels(): string[]` - 获取支持的模型列表
- `getModelInfo(modelId: string)` - 获取模型详细信息
### 类型定义
```typescript
interface EmbedResult {
vec: number[]; // 嵌入向量
tokens: number; // token 数量
embed_input?: string; // 原始输入文本
}
interface ModelLoadResult {
model_loaded: boolean; // 是否加载成功
}
interface ModelUnloadResult {
model_unloaded: boolean; // 是否卸载成功
}
interface TokenCountResult {
tokens: number; // token 数量
}
```
## 错误处理
```typescript
try {
await embeddingManager.loadModel('invalid-model');
} catch (error) {
console.error('加载模型失败:', error.message);
}
try {
const result = await embeddingManager.embed('');
} catch (error) {
console.error('文本不能为空:', error.message);
}
```
## 性能考虑
1. **模型加载**: 首次加载模型需要下载和初始化,可能需要几秒到几分钟
2. **批处理**: 使用 `embedBatch` 比多次调用 `embed` 更高效
3. **内存使用**: 大模型需要更多内存,注意设备限制
4. **GPU 加速**: 在支持 WebGPU 的浏览器中可以启用 GPU 加速
## 注意事项
- 首次使用某个模型时需要从 Hugging Face 下载,请确保网络连接正常
- 模型文件会被浏览器缓存,后续使用会更快
- 在移动设备上使用大模型可能会遇到内存限制
- Worker 在后台运行,不会阻塞 UI 线程

View File

@ -0,0 +1,353 @@
// 完整的嵌入 Worker使用 Transformers.js
console.log('Embedding worker loaded');
// 类型定义
interface EmbedInput {
embed_input: string;
}
interface EmbedResult {
vec: number[];
tokens: number;
embed_input?: string;
}
interface WorkerMessage {
method: string;
params: any;
id: number;
worker_id?: string;
}
interface WorkerResponse {
id: number;
result?: any;
error?: string;
worker_id?: string;
}
// 全局变量
let model: any = null;
let pipeline: any = null;
let tokenizer: any = null;
let processing_message = false;
let transformersLoaded = false;
// 动态导入 Transformers.js
async function loadTransformers() {
if (transformersLoaded) return;
try {
console.log('Loading Transformers.js...');
// 尝试使用旧版本的 Transformers.js它在 Worker 中更稳定
const { pipeline: pipelineFactory, env, AutoTokenizer } = await import('@xenova/transformers');
// 配置环境以适应浏览器 Worker
env.allowLocalModels = false;
env.allowRemoteModels = true;
// 配置 WASM 后端
env.backends.onnx.wasm.numThreads = 2; // 在 Worker 中使用单线程
env.backends.onnx.wasm.simd = true;
// 禁用 Node.js 特定功能
env.useFS = false;
env.useBrowserCache = true;
// 存储导入的函数
(globalThis as any).pipelineFactory = pipelineFactory;
(globalThis as any).AutoTokenizer = AutoTokenizer;
(globalThis as any).env = env;
transformersLoaded = true;
console.log('Transformers.js loaded successfully');
} catch (error) {
console.error('Failed to load Transformers.js:', error);
throw new Error(`Failed to load Transformers.js: ${error}`);
}
}
// 加载模型
async function loadModel(modelKey: string, useGpu: boolean = false) {
try {
console.log(`Loading model: ${modelKey}, GPU: ${useGpu}`);
// 确保 Transformers.js 已加载
await loadTransformers();
const pipelineFactory = (globalThis as any).pipelineFactory;
const AutoTokenizer = (globalThis as any).AutoTokenizer;
const env = (globalThis as any).env;
// 配置管道选项
const pipelineOpts: any = {
quantized: true,
progress_callback: (progress: any) => {
console.log('Model loading progress:', progress);
}
};
if (useGpu && typeof navigator !== 'undefined' && 'gpu' in navigator) {
console.log('[Transformers] Attempting to use GPU');
try {
pipelineOpts.device = 'webgpu';
pipelineOpts.dtype = 'fp32';
} catch (error) {
console.warn('[Transformers] GPU not available, falling back to CPU');
}
} else {
console.log('[Transformers] Using CPU');
}
// 创建嵌入管道
pipeline = await pipelineFactory('feature-extraction', modelKey, pipelineOpts);
// 创建分词器
tokenizer = await AutoTokenizer.from_pretrained(modelKey);
model = {
loaded: true,
model_key: modelKey,
use_gpu: useGpu
};
console.log(`Model ${modelKey} loaded successfully`);
return { model_loaded: true };
} catch (error) {
console.error('Error loading model:', error);
throw new Error(`Failed to load model: ${error}`);
}
}
// 卸载模型
async function unloadModel() {
try {
console.log('Unloading model...');
if (pipeline) {
if (pipeline.destroy) {
pipeline.destroy();
}
pipeline = null;
}
if (tokenizer) {
tokenizer = null;
}
model = null;
console.log('Model unloaded successfully');
return { model_unloaded: true };
} catch (error) {
console.error('Error unloading model:', error);
throw new Error(`Failed to unload model: ${error}`);
}
}
// 计算 token 数量
async function countTokens(input: string) {
try {
if (!tokenizer) {
throw new Error('Tokenizer not loaded');
}
const { input_ids } = await tokenizer(input);
return { tokens: input_ids.data.length };
} catch (error) {
console.error('Error counting tokens:', error);
throw new Error(`Failed to count tokens: ${error}`);
}
}
// 生成嵌入向量
async function embedBatch(inputs: EmbedInput[]): Promise<EmbedResult[]> {
try {
if (!pipeline || !tokenizer) {
throw new Error('Model not loaded');
}
console.log(`Processing ${inputs.length} inputs`);
// 过滤空输入
const filteredInputs = inputs.filter(item => item.embed_input && item.embed_input.length > 0);
if (filteredInputs.length === 0) {
return [];
}
// 批处理大小(可以根据需要调整)
const batchSize = 1;
if (filteredInputs.length > batchSize) {
console.log(`Processing ${filteredInputs.length} inputs in batches of ${batchSize}`);
const results: EmbedResult[] = [];
for (let i = 0; i < filteredInputs.length; i += batchSize) {
const batch = filteredInputs.slice(i, i + batchSize);
const batchResults = await processBatch(batch);
results.push(...batchResults);
}
return results;
}
return await processBatch(filteredInputs);
} catch (error) {
console.error('Error in embed batch:', error);
throw new Error(`Failed to generate embeddings: ${error}`);
}
}
// 处理单个批次
async function processBatch(batchInputs: EmbedInput[]): Promise<EmbedResult[]> {
try {
// 计算每个输入的 token 数量
const tokens = await Promise.all(
batchInputs.map(item => countTokens(item.embed_input))
);
// 准备嵌入输入(处理超长文本)
const maxTokens = 512; // 大多数模型的最大 token 限制
const embedInputs = await Promise.all(
batchInputs.map(async (item, i) => {
if (tokens[i].tokens < maxTokens) {
return item.embed_input;
}
// 截断超长文本
let tokenCt = tokens[i].tokens;
let truncatedInput = item.embed_input;
while (tokenCt > maxTokens) {
const pct = maxTokens / tokenCt;
const maxChars = Math.floor(truncatedInput.length * pct * 0.9);
truncatedInput = truncatedInput.substring(0, maxChars) + '...';
tokenCt = (await countTokens(truncatedInput)).tokens;
}
tokens[i].tokens = tokenCt;
return truncatedInput;
})
);
// 生成嵌入向量
const resp = await pipeline(embedInputs, { pooling: 'mean', normalize: true });
// 处理结果
return batchInputs.map((item, i) => ({
vec: Array.from(resp[i].data).map((val: number) => Math.round(val * 1e8) / 1e8),
tokens: tokens[i].tokens,
embed_input: item.embed_input
}));
} catch (error) {
console.error('Error processing batch:', error);
// 如果批处理失败,尝试逐个处理
return Promise.all(
batchInputs.map(async (item) => {
try {
const result = await pipeline(item.embed_input, { pooling: 'mean', normalize: true });
const tokenCount = await countTokens(item.embed_input);
return {
vec: Array.from(result[0].data).map((val: number) => Math.round(val * 1e8) / 1e8),
tokens: tokenCount.tokens,
embed_input: item.embed_input
};
} catch (singleError) {
console.error('Error processing single item:', singleError);
return {
vec: [],
tokens: 0,
embed_input: item.embed_input,
error: (singleError as Error).message
} as any;
}
})
);
}
}
// 处理消息
async function processMessage(data: WorkerMessage): Promise<WorkerResponse> {
const { method, params, id, worker_id } = data;
try {
let result: any;
switch (method) {
case 'load':
console.log('Load method called with params:', params);
result = await loadModel(params.model_key, params.use_gpu || false);
break;
case 'unload':
console.log('Unload method called');
result = await unloadModel();
break;
case 'embed_batch':
console.log('Embed batch method called');
if (!model) {
throw new Error('Model not loaded');
}
// 等待之前的处理完成
if (processing_message) {
while (processing_message) {
await new Promise(resolve => setTimeout(resolve, 100));
}
}
processing_message = true;
result = await embedBatch(params.inputs);
processing_message = false;
break;
case 'count_tokens':
console.log('Count tokens method called');
if (!model) {
throw new Error('Model not loaded');
}
// 等待之前的处理完成
if (processing_message) {
while (processing_message) {
await new Promise(resolve => setTimeout(resolve, 100));
}
}
processing_message = true;
result = await countTokens(params);
processing_message = false;
break;
default:
throw new Error(`Unknown method: ${method}`);
}
return { id, result, worker_id };
} catch (error) {
console.error('Error processing message:', error);
processing_message = false;
return { id, error: (error as Error).message, worker_id };
}
}
// 监听消息
self.addEventListener('message', async (event) => {
console.log('Worker received message:', event.data);
const response = await processMessage(event.data);
console.log('Worker sending response:', response);
self.postMessage(response);
});
console.log('Embedding worker ready');

15
src/embedworker/index.ts Normal file
View File

@ -0,0 +1,15 @@
import { EmbeddingManager } from "./EmbeddingManager";
// 创建一个单例的 Manager以便在整个应用中共享同一个 Worker
export const embeddingManager = new EmbeddingManager();
// 导出 EmbeddingManager 类以便其他地方使用
export { EmbeddingManager };
// 导出类型定义
export type {
EmbedResult,
ModelLoadResult,
ModelUnloadResult,
TokenCountResult
} from './EmbeddingManager';

View File

@ -4,8 +4,10 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
import { editorStateToPlainText } from '../components/chat-view/chat-input/utils/editor-state-to-plain-text' import { editorStateToPlainText } from '../components/chat-view/chat-input/utils/editor-state-to-plain-text'
import { useApp } from '../contexts/AppContext' import { useApp } from '../contexts/AppContext'
import { useSettings } from '../contexts/SettingsContext'
import { ChatManager } from '../database/json/chat/ChatManager' import { ChatManager } from '../database/json/chat/ChatManager'
import { deserializeChatMessage, serializeChatMessage } from '../database/json/utils' import { deserializeChatMessage, serializeChatMessage } from '../database/json/utils'
import { WorkspaceManager } from '../database/json/workspace/WorkspaceManager'
import { ChatConversationMeta, ChatMessage, ChatUserMessage } from '../types/chat' import { ChatConversationMeta, ChatMessage, ChatUserMessage } from '../types/chat'
type UseChatHistory = { type UseChatHistory = {
@ -22,7 +24,9 @@ type UseChatHistory = {
export function useChatHistory(): UseChatHistory { export function useChatHistory(): UseChatHistory {
const app = useApp() const app = useApp()
const chatManager = useMemo(() => new ChatManager(app), [app]) const { settings } = useSettings()
const workspaceManager = useMemo(() => new WorkspaceManager(app), [app])
const chatManager = useMemo(() => new ChatManager(app, workspaceManager), [app, workspaceManager])
const [chatList, setChatList] = useState<ChatConversationMeta[]>([]) const [chatList, setChatList] = useState<ChatConversationMeta[]>([])
@ -31,6 +35,9 @@ export function useChatHistory(): UseChatHistory {
setChatList(conversations) setChatList(conversations)
}, [chatManager]) }, [chatManager])
// 获取当前工作区
const currentWorkspace = settings.workspace || 'vault'
useEffect(() => { useEffect(() => {
void fetchChatList() void fetchChatList()
}, [fetchChatList]) }, [fetchChatList])
@ -52,7 +59,7 @@ export function useChatHistory(): UseChatHistory {
} else { } else {
const firstUserMessage = messages.find((v) => v.role === 'user') as ChatUserMessage const firstUserMessage = messages.find((v) => v.role === 'user') as ChatUserMessage
const newChat = await chatManager.createChat({ await chatManager.createChat({
id, id,
title: firstUserMessage?.content title: firstUserMessage?.content
? editorStateToPlainText(firstUserMessage.content).substring( ? editorStateToPlainText(firstUserMessage.content).substring(
@ -61,6 +68,7 @@ export function useChatHistory(): UseChatHistory {
) )
: 'New chat', : 'New chat',
messages: serializedMessages, messages: serializedMessages,
workspace: currentWorkspace,
}) })
} }
@ -71,7 +79,7 @@ export function useChatHistory(): UseChatHistory {
maxWait: 1000, maxWait: 1000,
}, },
), ),
[chatManager, fetchChatList], [chatManager, fetchChatList, settings, workspaceManager],
) )
const deleteConversation = useCallback( const deleteConversation = useCallback(

View File

@ -1,7 +1,7 @@
// @ts-nocheck // @ts-nocheck
import { EditorView } from '@codemirror/view' import { EditorView } from '@codemirror/view'
// import { PGlite } from '@electric-sql/pglite' // import { PGlite } from '@electric-sql/pglite'
import { App, Editor, MarkdownView, Modal, Notice, Plugin, TFile } from 'obsidian' import { Editor, MarkdownView, Modal, Notice, Plugin, TFile } from 'obsidian'
import { ApplyView } from './ApplyView' import { ApplyView } from './ApplyView'
import { ChatView } from './ChatView' import { ChatView } from './ChatView'
@ -388,7 +388,7 @@ export default class InfioPlugin extends Plugin {
name: '测试 Dataview简单查询', name: '测试 Dataview简单查询',
callback: async () => { callback: async () => {
console.log('开始测试 Dataview...'); console.log('开始测试 Dataview...');
if (!this.dataviewManager) { if (!this.dataviewManager) {
new Notice('DataviewManager 未初始化'); new Notice('DataviewManager 未初始化');
return; return;
@ -401,11 +401,11 @@ export default class InfioPlugin extends Plugin {
} }
console.log('Dataview API 可用,执行简单查询...'); console.log('Dataview API 可用,执行简单查询...');
try { try {
// 执行一个最简单的查询 // 执行一个最简单的查询
const result = await this.dataviewManager.executeQuery('LIST FROM ""'); const result = await this.dataviewManager.executeQuery('LIST FROM ""');
if (result.success) { if (result.success) {
new Notice('Dataview 查询成功!结果已在控制台输出'); new Notice('Dataview 查询成功!结果已在控制台输出');
console.log('查询结果:', result.data); console.log('查询结果:', result.data);
@ -419,6 +419,47 @@ export default class InfioPlugin extends Plugin {
} }
}, },
}); });
// 添加本地嵌入测试命令
this.addCommand({
id: 'test-local-embed',
name: '测试本地嵌入模型',
callback: async () => {
try {
// 动态导入嵌入管理器
const { embeddingManager } = await import('./embedworker/index');
// 加载模型
await embeddingManager.loadModel("Xenova/all-MiniLM-L6-v2", true);
// 测试嵌入 "hello world"
const testText = "hello world";
const result = await embeddingManager.embed(testText);
// 显示结果
const resultMessage = `
: "${testText}"
Token 数量: ${result.tokens}
向量维度: ${result.vec.length}
4: [${result.vec.slice(0, 4).map(v => v.toFixed(4)).join(', ')}...]
`.trim();
console.log('本地嵌入测试结果:', result);
// 创建模态框显示结果
const modal = new Modal(this.app);
modal.titleEl.setText('本地嵌入测试结果');
modal.contentEl.createEl('pre', { text: resultMessage });
modal.open();
} catch (error) {
console.error('嵌入测试失败:', error);
new Notice(`嵌入测试失败: ${error.message}`, 5000);
}
},
});
} }
onunload() { onunload() {

View File

@ -81,4 +81,5 @@ export type ChatConversationMeta = {
title: string title: string
createdAt: number createdAt: number
updatedAt: number updatedAt: number
workspace?: string
} }

View File

@ -1,4 +1,4 @@
// @ts-expect-error - parse5 and JSON5 types are not perfectly aligned with the dynamic parsing logic // @ts-nocheck
import JSON5 from 'json5' import JSON5 from 'json5'
import { parseFragment } from 'parse5' import { parseFragment } from 'parse5'

View File

@ -71,5 +71,5 @@
// } // }
}, },
// Specify which files to include in compilation // Specify which files to include in compilation
"include": ["src/**/*.ts", "src/**/*.tsx", "__mocks__"] "include": ["src/**/*.ts", "src/**/*.tsx", "__mocks__", "src/embedworker/*.worker.ts", "src/embedworker/*.worker.js"]
} }