update files search methods && semantic search mehtod

This commit is contained in:
duanfuxiang 2025-03-31 17:33:53 +08:00
parent d72c871716
commit 1e85149660
15 changed files with 87 additions and 28 deletions

View File

@ -27,7 +27,7 @@ import {
LLMBaseUrlNotSetException,
LLMModelNotSetException,
} from '../../core/llm/exception'
import { regexSearchFiles } from '../../core/services/ripgrep'
import { regexSearchFiles } from '../../core/ripgrep'
import { useChatHistory } from '../../hooks/use-chat-history'
import { ApplyStatus, ToolArgs } from '../../types/apply'
import { ChatMessage, ChatUserMessage } from '../../types/chat'
@ -527,10 +527,9 @@ const Chat = forwardRef<ChatRef, ChatProps>((props, ref) => {
}
} else if (toolArgs.type === 'regex_search_files') {
const baseVaultPath = app.vault.adapter.getBasePath()
const ripgrepPath = settings.ripgrepPath
const absolutePath = path.join(baseVaultPath, toolArgs.filepath)
console.log("absolutePath", absolutePath)
const results = await regexSearchFiles(absolutePath, toolArgs.regex)
console.log("results", results)
const results = await regexSearchFiles(absolutePath, toolArgs.regex, ripgrepPath)
const formattedContent = `[regex_search_files for '${toolArgs.filepath}'] Result:\n${results}\n`;
return {
type: 'regex_search_files',

View File

@ -1,4 +1,4 @@
import { Check, Edit, Loader2, X, Diff } from 'lucide-react'
import { Check, Diff, Loader2, X } from 'lucide-react'
import { PropsWithChildren, useState } from 'react'
import { useDarkModeContext } from '../../contexts/DarkModeContext'

View File

@ -137,7 +137,7 @@ const MarkdownWithIcons = ({
{markdownContent}
</ReactMarkdown>
</div>
{markdownContent && finish &&
{markdownContent && finish && iconName === "attempt_completion" &&
<div className="infio-chat-message-actions">
<CopyButton message={markdownContent} />
<CreateNewFileButton message={markdownContent} />

View File

@ -17,7 +17,7 @@ import { GroqProvider } from './groq'
import { InfioProvider } from './infio'
import { OllamaProvider } from './ollama'
import { OpenAIAuthenticatedProvider } from './openai'
import { OpenAICompatibleProvider } from './openai-compatible-provider'
import { OpenAICompatibleProvider } from './openai-compatible'
export type LLMManagerInterface = {

View File

@ -1,3 +1,6 @@
import https from 'https'
import { URL } from 'url'
import { GoogleGenerativeAI } from '@google/generative-ai'
import { OpenAI } from 'openai'
@ -176,6 +179,42 @@ export const getEmbeddingModel = (
},
}
}
case ApiProvider.OpenAICompatible: {
const openai = new OpenAI({
apiKey: settings.openaicompatibleProvider.apiKey,
baseURL: settings.openaicompatibleProvider.baseUrl,
dangerouslyAllowBrowser: true,
});
return {
id: settings.embeddingModelId,
dimension: 0,
getEmbedding: async (text: string) => {
try {
if (!openai.apiKey) {
throw new LLMAPIKeyNotSetException(
'OpenAI Compatible API key is missing. Please set it in settings menu.',
)
}
const embedding = await openai.embeddings.create({
model: settings.embeddingModelId,
input: text,
encoding_format: "float",
})
return embedding.data[0].embedding
} catch (error) {
if (
error.status === 429 &&
error.message.toLowerCase().includes('rate limit')
) {
throw new LLMRateLimitExceededException(
'OpenAI Compatible API rate limit exceeded. Please try again later.',
)
}
throw error
}
},
}
}
default:
throw new Error('Invalid embedding model')
}

View File

@ -34,7 +34,8 @@ export class RAGEngine {
}
async initializeDimension(): Promise<void> {
if (this.embeddingModel.dimension === 0 && this.settings.embeddingModelProvider === ApiProvider.Ollama) {
if (this.embeddingModel.dimension === 0 &&
(this.settings.embeddingModelProvider === ApiProvider.Ollama || this.settings.embeddingModelProvider === ApiProvider.OpenAICompatible)) {
this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length
}
}

View File

@ -30,8 +30,8 @@ export function truncateLine(line: string, maxLength: number = MAX_LINE_LENGTH):
return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line
}
async function getBinPath(): Promise<string | undefined> {
const binPath = path.join("/opt/homebrew/bin/", binName)
async function getBinPath(ripgrepPath: string): Promise<string | undefined> {
const binPath = path.join(ripgrepPath, binName)
return (await pathExists(binPath)) ? binPath : undefined
}
@ -86,20 +86,21 @@ async function execRipgrep(bin: string, args: string[]): Promise<string> {
export async function regexSearchFiles(
directoryPath: string,
regex: string,
ripgrepPath: string,
): Promise<string> {
const rgPath = await getBinPath()
const rgPath = await getBinPath(ripgrepPath)
if (!rgPath) {
throw new Error("Could not find ripgrep binary")
}
// 使用--glob参数排除.obsidian目录
// use --glob param to exclude .obsidian directory
const args = [
"--json",
"-e",
regex,
"--glob",
"!.obsidian/**", // 排除.obsidian目录及其所有子目录
"!.obsidian/**", // exclude .obsidian directory and all its subdirectories
"--glob",
"!.git/**",
"--context",

View File

@ -62,7 +62,6 @@ export class ConversationRepository {
message.createdAt || new Date()
]
)
console.log('createMessage: ', message.id, result)
return result.rows[0]
}
@ -129,11 +128,10 @@ export class ConversationRepository {
}
async deleteAllMessagesFromConversation(conversationId: string, tx?: Transaction): Promise<void> {
const result = await (tx ?? this.db).query(
await (tx ?? this.db).query(
`DELETE FROM messages WHERE conversation_id = $1`,
[conversationId]
)
console.log('deleteAllMessagesFromConversation', conversationId, result)
return
}
}

View File

@ -142,8 +142,8 @@ export default class InfioPlugin extends Plugin {
this.app.metadataCache.on("changed", (file: TFile) => {
if (file) {
eventListener.handleFileChange(file);
console.log("file changed: filename: ", file.name);
this.ragEngine?.updateFileIndex(file);
// is not worth it to update the file index on every file change
// this.ragEngine?.updateFileIndex(file);
}
})
);
@ -151,7 +151,6 @@ export default class InfioPlugin extends Plugin {
this.registerEvent(
this.app.metadataCache.on("deleted", (file: TFile) => {
if (file) {
console.log("file deleted: filename: ", file.name)
this.ragEngine?.deleteFileIndex(file);
}
})

View File

@ -37,8 +37,8 @@ export class InfioSettingTab extends PluginSettingTab {
const { containerEl } = this
containerEl.empty()
this.renderModelsSection(containerEl)
this.renderDeepResearchSection(containerEl)
this.renderFilesSearchSection(containerEl)
this.renderDeepResearchSection(containerEl)
this.renderRAGSection(containerEl)
this.renderAutoCompleteSection(containerEl)
}
@ -60,15 +60,15 @@ export class InfioSettingTab extends PluginSettingTab {
}
private renderFilesSearchSection(containerEl: HTMLElement): void {
new Setting(containerEl).setHeading().setName('File search')
new Setting(containerEl)
.setHeading()
.setName('Files Search Method')
.setName('Files search method')
.setDesc('Choose the method to search for files.')
.addDropdown((dropdown) =>
dropdown
.addOption('auto', 'Auto')
.addOption('regex', 'Regex')
.addOption('semantic', 'Semantic')
.addOption('regex', 'Regex')
.setValue(this.plugin.settings.filesSearchMethod)
.onChange(async (value) => {
await this.plugin.setSettings({
@ -77,6 +77,20 @@ export class InfioSettingTab extends PluginSettingTab {
})
}),
)
new Setting(containerEl)
.setName('ripgrep path')
.setDesc('Path to the ripgrep binary. When using regex search, this is required.')
.addText((text) =>
text
.setPlaceholder('/opt/homebrew/bin/')
.setValue(this.plugin.settings.ripgrepPath)
.onChange(async (value) => {
await this.plugin.setSettings({
...this.plugin.settings,
ripgrepPath: value,
})
}),
)
}
renderModelsSection(containerEl: HTMLElement): void {
@ -88,10 +102,10 @@ export class InfioSettingTab extends PluginSettingTab {
renderDeepResearchSection(containerEl: HTMLElement): void {
new Setting(containerEl)
.setHeading()
.setName('Deep Research')
.setName('Deep research')
new Setting(containerEl)
.setName('Serper Api Key')
.setName('Serper API key')
.setDesc(createFragment(el => {
el.appendText('API key for web search functionality. Serper allows the plugin to search the internet for information, similar to a search engine. Get your key from ');
const a = el.createEl('a', {
@ -114,7 +128,7 @@ export class InfioSettingTab extends PluginSettingTab {
)
new Setting(containerEl)
.setName('Jina Api Key (Optional)')
.setName('Jina API key (Optional)')
.setDesc(createFragment(el => {
el.appendText('API key for parsing web pages into markdown format. If not provided, local parsing will be used. Get your key from ');
const a = el.createEl('a', {

View File

@ -50,7 +50,7 @@ const CustomProviderSettings: React.FC<CustomProviderSettingsProps> = ({ plugin,
const handleSettingsUpdate = async (newSettings: InfioSettings) => {
await plugin.setSettings(newSettings);
// 使用父组件传入的回调函数来刷新整个容器
// Use the callback function passed from the parent component to refresh the entire container
onSettingsUpdate?.();
};

View File

@ -237,6 +237,7 @@ export const InfioSettingsSchema = z.object({
// Files Search
filesSearchMethod: z.enum(['regex', 'semantic', 'auto']).catch('auto'),
ripgrepPath: z.string().catch(''),
/// [compatible]
// activeModels [compatible]

View File

@ -467,7 +467,14 @@ export class PromptGenerator {
}
private async getSystemMessageNew(mode: Mode, filesSearchMethod: string, preferredLanguage: string): Promise<RequestMessage> {
const systemPrompt = await SYSTEM_PROMPT(this.app.vault.getRoot().path, false, mode, filesSearchMethod, preferredLanguage, this.diffStrategy)
const systemPrompt = await SYSTEM_PROMPT(
this.app.vault.getRoot().path,
false,
mode,
filesSearchMethod,
preferredLanguage,
this.diffStrategy
)
return {
role: 'system',