update files search methods && semantic search mehtod

This commit is contained in:
duanfuxiang 2025-03-31 17:33:53 +08:00
parent d72c871716
commit 1e85149660
15 changed files with 87 additions and 28 deletions

View File

@ -27,7 +27,7 @@ import {
LLMBaseUrlNotSetException, LLMBaseUrlNotSetException,
LLMModelNotSetException, LLMModelNotSetException,
} from '../../core/llm/exception' } from '../../core/llm/exception'
import { regexSearchFiles } from '../../core/services/ripgrep' import { regexSearchFiles } from '../../core/ripgrep'
import { useChatHistory } from '../../hooks/use-chat-history' import { useChatHistory } from '../../hooks/use-chat-history'
import { ApplyStatus, ToolArgs } from '../../types/apply' import { ApplyStatus, ToolArgs } from '../../types/apply'
import { ChatMessage, ChatUserMessage } from '../../types/chat' import { ChatMessage, ChatUserMessage } from '../../types/chat'
@ -527,10 +527,9 @@ const Chat = forwardRef<ChatRef, ChatProps>((props, ref) => {
} }
} else if (toolArgs.type === 'regex_search_files') { } else if (toolArgs.type === 'regex_search_files') {
const baseVaultPath = app.vault.adapter.getBasePath() const baseVaultPath = app.vault.adapter.getBasePath()
const ripgrepPath = settings.ripgrepPath
const absolutePath = path.join(baseVaultPath, toolArgs.filepath) const absolutePath = path.join(baseVaultPath, toolArgs.filepath)
console.log("absolutePath", absolutePath) const results = await regexSearchFiles(absolutePath, toolArgs.regex, ripgrepPath)
const results = await regexSearchFiles(absolutePath, toolArgs.regex)
console.log("results", results)
const formattedContent = `[regex_search_files for '${toolArgs.filepath}'] Result:\n${results}\n`; const formattedContent = `[regex_search_files for '${toolArgs.filepath}'] Result:\n${results}\n`;
return { return {
type: 'regex_search_files', type: 'regex_search_files',

View File

@ -1,4 +1,4 @@
import { Check, Edit, Loader2, X, Diff } from 'lucide-react' import { Check, Diff, Loader2, X } from 'lucide-react'
import { PropsWithChildren, useState } from 'react' import { PropsWithChildren, useState } from 'react'
import { useDarkModeContext } from '../../contexts/DarkModeContext' import { useDarkModeContext } from '../../contexts/DarkModeContext'

View File

@ -137,7 +137,7 @@ const MarkdownWithIcons = ({
{markdownContent} {markdownContent}
</ReactMarkdown> </ReactMarkdown>
</div> </div>
{markdownContent && finish && {markdownContent && finish && iconName === "attempt_completion" &&
<div className="infio-chat-message-actions"> <div className="infio-chat-message-actions">
<CopyButton message={markdownContent} /> <CopyButton message={markdownContent} />
<CreateNewFileButton message={markdownContent} /> <CreateNewFileButton message={markdownContent} />

View File

@ -17,7 +17,7 @@ import { GroqProvider } from './groq'
import { InfioProvider } from './infio' import { InfioProvider } from './infio'
import { OllamaProvider } from './ollama' import { OllamaProvider } from './ollama'
import { OpenAIAuthenticatedProvider } from './openai' import { OpenAIAuthenticatedProvider } from './openai'
import { OpenAICompatibleProvider } from './openai-compatible-provider' import { OpenAICompatibleProvider } from './openai-compatible'
export type LLMManagerInterface = { export type LLMManagerInterface = {

View File

@ -1,3 +1,6 @@
import https from 'https'
import { URL } from 'url'
import { GoogleGenerativeAI } from '@google/generative-ai' import { GoogleGenerativeAI } from '@google/generative-ai'
import { OpenAI } from 'openai' import { OpenAI } from 'openai'
@ -176,6 +179,42 @@ export const getEmbeddingModel = (
}, },
} }
} }
case ApiProvider.OpenAICompatible: {
const openai = new OpenAI({
apiKey: settings.openaicompatibleProvider.apiKey,
baseURL: settings.openaicompatibleProvider.baseUrl,
dangerouslyAllowBrowser: true,
});
return {
id: settings.embeddingModelId,
dimension: 0,
getEmbedding: async (text: string) => {
try {
if (!openai.apiKey) {
throw new LLMAPIKeyNotSetException(
'OpenAI Compatible API key is missing. Please set it in settings menu.',
)
}
const embedding = await openai.embeddings.create({
model: settings.embeddingModelId,
input: text,
encoding_format: "float",
})
return embedding.data[0].embedding
} catch (error) {
if (
error.status === 429 &&
error.message.toLowerCase().includes('rate limit')
) {
throw new LLMRateLimitExceededException(
'OpenAI Compatible API rate limit exceeded. Please try again later.',
)
}
throw error
}
},
}
}
default: default:
throw new Error('Invalid embedding model') throw new Error('Invalid embedding model')
} }

View File

@ -34,7 +34,8 @@ export class RAGEngine {
} }
async initializeDimension(): Promise<void> { async initializeDimension(): Promise<void> {
if (this.embeddingModel.dimension === 0 && this.settings.embeddingModelProvider === ApiProvider.Ollama) { if (this.embeddingModel.dimension === 0 &&
(this.settings.embeddingModelProvider === ApiProvider.Ollama || this.settings.embeddingModelProvider === ApiProvider.OpenAICompatible)) {
this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length
} }
} }

View File

@ -30,8 +30,8 @@ export function truncateLine(line: string, maxLength: number = MAX_LINE_LENGTH):
return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line
} }
async function getBinPath(): Promise<string | undefined> { async function getBinPath(ripgrepPath: string): Promise<string | undefined> {
const binPath = path.join("/opt/homebrew/bin/", binName) const binPath = path.join(ripgrepPath, binName)
return (await pathExists(binPath)) ? binPath : undefined return (await pathExists(binPath)) ? binPath : undefined
} }
@ -86,20 +86,21 @@ async function execRipgrep(bin: string, args: string[]): Promise<string> {
export async function regexSearchFiles( export async function regexSearchFiles(
directoryPath: string, directoryPath: string,
regex: string, regex: string,
ripgrepPath: string,
): Promise<string> { ): Promise<string> {
const rgPath = await getBinPath() const rgPath = await getBinPath(ripgrepPath)
if (!rgPath) { if (!rgPath) {
throw new Error("Could not find ripgrep binary") throw new Error("Could not find ripgrep binary")
} }
// 使用--glob参数排除.obsidian目录 // use --glob param to exclude .obsidian directory
const args = [ const args = [
"--json", "--json",
"-e", "-e",
regex, regex,
"--glob", "--glob",
"!.obsidian/**", // 排除.obsidian目录及其所有子目录 "!.obsidian/**", // exclude .obsidian directory and all its subdirectories
"--glob", "--glob",
"!.git/**", "!.git/**",
"--context", "--context",

View File

@ -62,7 +62,6 @@ export class ConversationRepository {
message.createdAt || new Date() message.createdAt || new Date()
] ]
) )
console.log('createMessage: ', message.id, result)
return result.rows[0] return result.rows[0]
} }
@ -129,11 +128,10 @@ export class ConversationRepository {
} }
async deleteAllMessagesFromConversation(conversationId: string, tx?: Transaction): Promise<void> { async deleteAllMessagesFromConversation(conversationId: string, tx?: Transaction): Promise<void> {
const result = await (tx ?? this.db).query( await (tx ?? this.db).query(
`DELETE FROM messages WHERE conversation_id = $1`, `DELETE FROM messages WHERE conversation_id = $1`,
[conversationId] [conversationId]
) )
console.log('deleteAllMessagesFromConversation', conversationId, result)
return return
} }
} }

View File

@ -142,8 +142,8 @@ export default class InfioPlugin extends Plugin {
this.app.metadataCache.on("changed", (file: TFile) => { this.app.metadataCache.on("changed", (file: TFile) => {
if (file) { if (file) {
eventListener.handleFileChange(file); eventListener.handleFileChange(file);
console.log("file changed: filename: ", file.name); // is not worth it to update the file index on every file change
this.ragEngine?.updateFileIndex(file); // this.ragEngine?.updateFileIndex(file);
} }
}) })
); );
@ -151,7 +151,6 @@ export default class InfioPlugin extends Plugin {
this.registerEvent( this.registerEvent(
this.app.metadataCache.on("deleted", (file: TFile) => { this.app.metadataCache.on("deleted", (file: TFile) => {
if (file) { if (file) {
console.log("file deleted: filename: ", file.name)
this.ragEngine?.deleteFileIndex(file); this.ragEngine?.deleteFileIndex(file);
} }
}) })

View File

@ -37,8 +37,8 @@ export class InfioSettingTab extends PluginSettingTab {
const { containerEl } = this const { containerEl } = this
containerEl.empty() containerEl.empty()
this.renderModelsSection(containerEl) this.renderModelsSection(containerEl)
this.renderDeepResearchSection(containerEl)
this.renderFilesSearchSection(containerEl) this.renderFilesSearchSection(containerEl)
this.renderDeepResearchSection(containerEl)
this.renderRAGSection(containerEl) this.renderRAGSection(containerEl)
this.renderAutoCompleteSection(containerEl) this.renderAutoCompleteSection(containerEl)
} }
@ -60,15 +60,15 @@ export class InfioSettingTab extends PluginSettingTab {
} }
private renderFilesSearchSection(containerEl: HTMLElement): void { private renderFilesSearchSection(containerEl: HTMLElement): void {
new Setting(containerEl).setHeading().setName('File search')
new Setting(containerEl) new Setting(containerEl)
.setHeading() .setName('Files search method')
.setName('Files Search Method')
.setDesc('Choose the method to search for files.') .setDesc('Choose the method to search for files.')
.addDropdown((dropdown) => .addDropdown((dropdown) =>
dropdown dropdown
.addOption('auto', 'Auto') .addOption('auto', 'Auto')
.addOption('regex', 'Regex')
.addOption('semantic', 'Semantic') .addOption('semantic', 'Semantic')
.addOption('regex', 'Regex')
.setValue(this.plugin.settings.filesSearchMethod) .setValue(this.plugin.settings.filesSearchMethod)
.onChange(async (value) => { .onChange(async (value) => {
await this.plugin.setSettings({ await this.plugin.setSettings({
@ -77,6 +77,20 @@ export class InfioSettingTab extends PluginSettingTab {
}) })
}), }),
) )
new Setting(containerEl)
.setName('ripgrep path')
.setDesc('Path to the ripgrep binary. When using regex search, this is required.')
.addText((text) =>
text
.setPlaceholder('/opt/homebrew/bin/')
.setValue(this.plugin.settings.ripgrepPath)
.onChange(async (value) => {
await this.plugin.setSettings({
...this.plugin.settings,
ripgrepPath: value,
})
}),
)
} }
renderModelsSection(containerEl: HTMLElement): void { renderModelsSection(containerEl: HTMLElement): void {
@ -88,10 +102,10 @@ export class InfioSettingTab extends PluginSettingTab {
renderDeepResearchSection(containerEl: HTMLElement): void { renderDeepResearchSection(containerEl: HTMLElement): void {
new Setting(containerEl) new Setting(containerEl)
.setHeading() .setHeading()
.setName('Deep Research') .setName('Deep research')
new Setting(containerEl) new Setting(containerEl)
.setName('Serper Api Key') .setName('Serper API key')
.setDesc(createFragment(el => { .setDesc(createFragment(el => {
el.appendText('API key for web search functionality. Serper allows the plugin to search the internet for information, similar to a search engine. Get your key from '); el.appendText('API key for web search functionality. Serper allows the plugin to search the internet for information, similar to a search engine. Get your key from ');
const a = el.createEl('a', { const a = el.createEl('a', {
@ -114,7 +128,7 @@ export class InfioSettingTab extends PluginSettingTab {
) )
new Setting(containerEl) new Setting(containerEl)
.setName('Jina Api Key (Optional)') .setName('Jina API key (Optional)')
.setDesc(createFragment(el => { .setDesc(createFragment(el => {
el.appendText('API key for parsing web pages into markdown format. If not provided, local parsing will be used. Get your key from '); el.appendText('API key for parsing web pages into markdown format. If not provided, local parsing will be used. Get your key from ');
const a = el.createEl('a', { const a = el.createEl('a', {

View File

@ -50,7 +50,7 @@ const CustomProviderSettings: React.FC<CustomProviderSettingsProps> = ({ plugin,
const handleSettingsUpdate = async (newSettings: InfioSettings) => { const handleSettingsUpdate = async (newSettings: InfioSettings) => {
await plugin.setSettings(newSettings); await plugin.setSettings(newSettings);
// 使用父组件传入的回调函数来刷新整个容器 // Use the callback function passed from the parent component to refresh the entire container
onSettingsUpdate?.(); onSettingsUpdate?.();
}; };

View File

@ -237,6 +237,7 @@ export const InfioSettingsSchema = z.object({
// Files Search // Files Search
filesSearchMethod: z.enum(['regex', 'semantic', 'auto']).catch('auto'), filesSearchMethod: z.enum(['regex', 'semantic', 'auto']).catch('auto'),
ripgrepPath: z.string().catch(''),
/// [compatible] /// [compatible]
// activeModels [compatible] // activeModels [compatible]

View File

@ -467,7 +467,14 @@ export class PromptGenerator {
} }
private async getSystemMessageNew(mode: Mode, filesSearchMethod: string, preferredLanguage: string): Promise<RequestMessage> { private async getSystemMessageNew(mode: Mode, filesSearchMethod: string, preferredLanguage: string): Promise<RequestMessage> {
const systemPrompt = await SYSTEM_PROMPT(this.app.vault.getRoot().path, false, mode, filesSearchMethod, preferredLanguage, this.diffStrategy) const systemPrompt = await SYSTEM_PROMPT(
this.app.vault.getRoot().path,
false,
mode,
filesSearchMethod,
preferredLanguage,
this.diffStrategy
)
return { return {
role: 'system', role: 'system',