update files search methods && semantic search mehtod
This commit is contained in:
parent
d72c871716
commit
1e85149660
@ -27,7 +27,7 @@ import {
|
|||||||
LLMBaseUrlNotSetException,
|
LLMBaseUrlNotSetException,
|
||||||
LLMModelNotSetException,
|
LLMModelNotSetException,
|
||||||
} from '../../core/llm/exception'
|
} from '../../core/llm/exception'
|
||||||
import { regexSearchFiles } from '../../core/services/ripgrep'
|
import { regexSearchFiles } from '../../core/ripgrep'
|
||||||
import { useChatHistory } from '../../hooks/use-chat-history'
|
import { useChatHistory } from '../../hooks/use-chat-history'
|
||||||
import { ApplyStatus, ToolArgs } from '../../types/apply'
|
import { ApplyStatus, ToolArgs } from '../../types/apply'
|
||||||
import { ChatMessage, ChatUserMessage } from '../../types/chat'
|
import { ChatMessage, ChatUserMessage } from '../../types/chat'
|
||||||
@ -527,10 +527,9 @@ const Chat = forwardRef<ChatRef, ChatProps>((props, ref) => {
|
|||||||
}
|
}
|
||||||
} else if (toolArgs.type === 'regex_search_files') {
|
} else if (toolArgs.type === 'regex_search_files') {
|
||||||
const baseVaultPath = app.vault.adapter.getBasePath()
|
const baseVaultPath = app.vault.adapter.getBasePath()
|
||||||
|
const ripgrepPath = settings.ripgrepPath
|
||||||
const absolutePath = path.join(baseVaultPath, toolArgs.filepath)
|
const absolutePath = path.join(baseVaultPath, toolArgs.filepath)
|
||||||
console.log("absolutePath", absolutePath)
|
const results = await regexSearchFiles(absolutePath, toolArgs.regex, ripgrepPath)
|
||||||
const results = await regexSearchFiles(absolutePath, toolArgs.regex)
|
|
||||||
console.log("results", results)
|
|
||||||
const formattedContent = `[regex_search_files for '${toolArgs.filepath}'] Result:\n${results}\n`;
|
const formattedContent = `[regex_search_files for '${toolArgs.filepath}'] Result:\n${results}\n`;
|
||||||
return {
|
return {
|
||||||
type: 'regex_search_files',
|
type: 'regex_search_files',
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import { Check, Edit, Loader2, X, Diff } from 'lucide-react'
|
import { Check, Diff, Loader2, X } from 'lucide-react'
|
||||||
import { PropsWithChildren, useState } from 'react'
|
import { PropsWithChildren, useState } from 'react'
|
||||||
|
|
||||||
import { useDarkModeContext } from '../../contexts/DarkModeContext'
|
import { useDarkModeContext } from '../../contexts/DarkModeContext'
|
||||||
|
|||||||
@ -137,7 +137,7 @@ const MarkdownWithIcons = ({
|
|||||||
{markdownContent}
|
{markdownContent}
|
||||||
</ReactMarkdown>
|
</ReactMarkdown>
|
||||||
</div>
|
</div>
|
||||||
{markdownContent && finish &&
|
{markdownContent && finish && iconName === "attempt_completion" &&
|
||||||
<div className="infio-chat-message-actions">
|
<div className="infio-chat-message-actions">
|
||||||
<CopyButton message={markdownContent} />
|
<CopyButton message={markdownContent} />
|
||||||
<CreateNewFileButton message={markdownContent} />
|
<CreateNewFileButton message={markdownContent} />
|
||||||
|
|||||||
@ -17,7 +17,7 @@ import { GroqProvider } from './groq'
|
|||||||
import { InfioProvider } from './infio'
|
import { InfioProvider } from './infio'
|
||||||
import { OllamaProvider } from './ollama'
|
import { OllamaProvider } from './ollama'
|
||||||
import { OpenAIAuthenticatedProvider } from './openai'
|
import { OpenAIAuthenticatedProvider } from './openai'
|
||||||
import { OpenAICompatibleProvider } from './openai-compatible-provider'
|
import { OpenAICompatibleProvider } from './openai-compatible'
|
||||||
|
|
||||||
|
|
||||||
export type LLMManagerInterface = {
|
export type LLMManagerInterface = {
|
||||||
|
|||||||
@ -1,3 +1,6 @@
|
|||||||
|
import https from 'https'
|
||||||
|
import { URL } from 'url'
|
||||||
|
|
||||||
import { GoogleGenerativeAI } from '@google/generative-ai'
|
import { GoogleGenerativeAI } from '@google/generative-ai'
|
||||||
import { OpenAI } from 'openai'
|
import { OpenAI } from 'openai'
|
||||||
|
|
||||||
@ -176,6 +179,42 @@ export const getEmbeddingModel = (
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
case ApiProvider.OpenAICompatible: {
|
||||||
|
const openai = new OpenAI({
|
||||||
|
apiKey: settings.openaicompatibleProvider.apiKey,
|
||||||
|
baseURL: settings.openaicompatibleProvider.baseUrl,
|
||||||
|
dangerouslyAllowBrowser: true,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
id: settings.embeddingModelId,
|
||||||
|
dimension: 0,
|
||||||
|
getEmbedding: async (text: string) => {
|
||||||
|
try {
|
||||||
|
if (!openai.apiKey) {
|
||||||
|
throw new LLMAPIKeyNotSetException(
|
||||||
|
'OpenAI Compatible API key is missing. Please set it in settings menu.',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const embedding = await openai.embeddings.create({
|
||||||
|
model: settings.embeddingModelId,
|
||||||
|
input: text,
|
||||||
|
encoding_format: "float",
|
||||||
|
})
|
||||||
|
return embedding.data[0].embedding
|
||||||
|
} catch (error) {
|
||||||
|
if (
|
||||||
|
error.status === 429 &&
|
||||||
|
error.message.toLowerCase().includes('rate limit')
|
||||||
|
) {
|
||||||
|
throw new LLMRateLimitExceededException(
|
||||||
|
'OpenAI Compatible API rate limit exceeded. Please try again later.',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
throw new Error('Invalid embedding model')
|
throw new Error('Invalid embedding model')
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,7 +34,8 @@ export class RAGEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async initializeDimension(): Promise<void> {
|
async initializeDimension(): Promise<void> {
|
||||||
if (this.embeddingModel.dimension === 0 && this.settings.embeddingModelProvider === ApiProvider.Ollama) {
|
if (this.embeddingModel.dimension === 0 &&
|
||||||
|
(this.settings.embeddingModelProvider === ApiProvider.Ollama || this.settings.embeddingModelProvider === ApiProvider.OpenAICompatible)) {
|
||||||
this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length
|
this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -30,8 +30,8 @@ export function truncateLine(line: string, maxLength: number = MAX_LINE_LENGTH):
|
|||||||
return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line
|
return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getBinPath(): Promise<string | undefined> {
|
async function getBinPath(ripgrepPath: string): Promise<string | undefined> {
|
||||||
const binPath = path.join("/opt/homebrew/bin/", binName)
|
const binPath = path.join(ripgrepPath, binName)
|
||||||
return (await pathExists(binPath)) ? binPath : undefined
|
return (await pathExists(binPath)) ? binPath : undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,20 +86,21 @@ async function execRipgrep(bin: string, args: string[]): Promise<string> {
|
|||||||
export async function regexSearchFiles(
|
export async function regexSearchFiles(
|
||||||
directoryPath: string,
|
directoryPath: string,
|
||||||
regex: string,
|
regex: string,
|
||||||
|
ripgrepPath: string,
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
const rgPath = await getBinPath()
|
const rgPath = await getBinPath(ripgrepPath)
|
||||||
|
|
||||||
if (!rgPath) {
|
if (!rgPath) {
|
||||||
throw new Error("Could not find ripgrep binary")
|
throw new Error("Could not find ripgrep binary")
|
||||||
}
|
}
|
||||||
|
|
||||||
// 使用--glob参数排除.obsidian目录
|
// use --glob param to exclude .obsidian directory
|
||||||
const args = [
|
const args = [
|
||||||
"--json",
|
"--json",
|
||||||
"-e",
|
"-e",
|
||||||
regex,
|
regex,
|
||||||
"--glob",
|
"--glob",
|
||||||
"!.obsidian/**", // 排除.obsidian目录及其所有子目录
|
"!.obsidian/**", // exclude .obsidian directory and all its subdirectories
|
||||||
"--glob",
|
"--glob",
|
||||||
"!.git/**",
|
"!.git/**",
|
||||||
"--context",
|
"--context",
|
||||||
@ -62,7 +62,6 @@ export class ConversationRepository {
|
|||||||
message.createdAt || new Date()
|
message.createdAt || new Date()
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
console.log('createMessage: ', message.id, result)
|
|
||||||
return result.rows[0]
|
return result.rows[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -129,11 +128,10 @@ export class ConversationRepository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async deleteAllMessagesFromConversation(conversationId: string, tx?: Transaction): Promise<void> {
|
async deleteAllMessagesFromConversation(conversationId: string, tx?: Transaction): Promise<void> {
|
||||||
const result = await (tx ?? this.db).query(
|
await (tx ?? this.db).query(
|
||||||
`DELETE FROM messages WHERE conversation_id = $1`,
|
`DELETE FROM messages WHERE conversation_id = $1`,
|
||||||
[conversationId]
|
[conversationId]
|
||||||
)
|
)
|
||||||
console.log('deleteAllMessagesFromConversation', conversationId, result)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -142,8 +142,8 @@ export default class InfioPlugin extends Plugin {
|
|||||||
this.app.metadataCache.on("changed", (file: TFile) => {
|
this.app.metadataCache.on("changed", (file: TFile) => {
|
||||||
if (file) {
|
if (file) {
|
||||||
eventListener.handleFileChange(file);
|
eventListener.handleFileChange(file);
|
||||||
console.log("file changed: filename: ", file.name);
|
// is not worth it to update the file index on every file change
|
||||||
this.ragEngine?.updateFileIndex(file);
|
// this.ragEngine?.updateFileIndex(file);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
@ -151,7 +151,6 @@ export default class InfioPlugin extends Plugin {
|
|||||||
this.registerEvent(
|
this.registerEvent(
|
||||||
this.app.metadataCache.on("deleted", (file: TFile) => {
|
this.app.metadataCache.on("deleted", (file: TFile) => {
|
||||||
if (file) {
|
if (file) {
|
||||||
console.log("file deleted: filename: ", file.name)
|
|
||||||
this.ragEngine?.deleteFileIndex(file);
|
this.ragEngine?.deleteFileIndex(file);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@ -37,8 +37,8 @@ export class InfioSettingTab extends PluginSettingTab {
|
|||||||
const { containerEl } = this
|
const { containerEl } = this
|
||||||
containerEl.empty()
|
containerEl.empty()
|
||||||
this.renderModelsSection(containerEl)
|
this.renderModelsSection(containerEl)
|
||||||
this.renderDeepResearchSection(containerEl)
|
|
||||||
this.renderFilesSearchSection(containerEl)
|
this.renderFilesSearchSection(containerEl)
|
||||||
|
this.renderDeepResearchSection(containerEl)
|
||||||
this.renderRAGSection(containerEl)
|
this.renderRAGSection(containerEl)
|
||||||
this.renderAutoCompleteSection(containerEl)
|
this.renderAutoCompleteSection(containerEl)
|
||||||
}
|
}
|
||||||
@ -60,15 +60,15 @@ export class InfioSettingTab extends PluginSettingTab {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private renderFilesSearchSection(containerEl: HTMLElement): void {
|
private renderFilesSearchSection(containerEl: HTMLElement): void {
|
||||||
|
new Setting(containerEl).setHeading().setName('File search')
|
||||||
new Setting(containerEl)
|
new Setting(containerEl)
|
||||||
.setHeading()
|
.setName('Files search method')
|
||||||
.setName('Files Search Method')
|
|
||||||
.setDesc('Choose the method to search for files.')
|
.setDesc('Choose the method to search for files.')
|
||||||
.addDropdown((dropdown) =>
|
.addDropdown((dropdown) =>
|
||||||
dropdown
|
dropdown
|
||||||
.addOption('auto', 'Auto')
|
.addOption('auto', 'Auto')
|
||||||
.addOption('regex', 'Regex')
|
|
||||||
.addOption('semantic', 'Semantic')
|
.addOption('semantic', 'Semantic')
|
||||||
|
.addOption('regex', 'Regex')
|
||||||
.setValue(this.plugin.settings.filesSearchMethod)
|
.setValue(this.plugin.settings.filesSearchMethod)
|
||||||
.onChange(async (value) => {
|
.onChange(async (value) => {
|
||||||
await this.plugin.setSettings({
|
await this.plugin.setSettings({
|
||||||
@ -77,6 +77,20 @@ export class InfioSettingTab extends PluginSettingTab {
|
|||||||
})
|
})
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
new Setting(containerEl)
|
||||||
|
.setName('ripgrep path')
|
||||||
|
.setDesc('Path to the ripgrep binary. When using regex search, this is required.')
|
||||||
|
.addText((text) =>
|
||||||
|
text
|
||||||
|
.setPlaceholder('/opt/homebrew/bin/')
|
||||||
|
.setValue(this.plugin.settings.ripgrepPath)
|
||||||
|
.onChange(async (value) => {
|
||||||
|
await this.plugin.setSettings({
|
||||||
|
...this.plugin.settings,
|
||||||
|
ripgrepPath: value,
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
renderModelsSection(containerEl: HTMLElement): void {
|
renderModelsSection(containerEl: HTMLElement): void {
|
||||||
@ -88,10 +102,10 @@ export class InfioSettingTab extends PluginSettingTab {
|
|||||||
renderDeepResearchSection(containerEl: HTMLElement): void {
|
renderDeepResearchSection(containerEl: HTMLElement): void {
|
||||||
new Setting(containerEl)
|
new Setting(containerEl)
|
||||||
.setHeading()
|
.setHeading()
|
||||||
.setName('Deep Research')
|
.setName('Deep research')
|
||||||
|
|
||||||
new Setting(containerEl)
|
new Setting(containerEl)
|
||||||
.setName('Serper Api Key')
|
.setName('Serper API key')
|
||||||
.setDesc(createFragment(el => {
|
.setDesc(createFragment(el => {
|
||||||
el.appendText('API key for web search functionality. Serper allows the plugin to search the internet for information, similar to a search engine. Get your key from ');
|
el.appendText('API key for web search functionality. Serper allows the plugin to search the internet for information, similar to a search engine. Get your key from ');
|
||||||
const a = el.createEl('a', {
|
const a = el.createEl('a', {
|
||||||
@ -114,7 +128,7 @@ export class InfioSettingTab extends PluginSettingTab {
|
|||||||
)
|
)
|
||||||
|
|
||||||
new Setting(containerEl)
|
new Setting(containerEl)
|
||||||
.setName('Jina Api Key (Optional)')
|
.setName('Jina API key (Optional)')
|
||||||
.setDesc(createFragment(el => {
|
.setDesc(createFragment(el => {
|
||||||
el.appendText('API key for parsing web pages into markdown format. If not provided, local parsing will be used. Get your key from ');
|
el.appendText('API key for parsing web pages into markdown format. If not provided, local parsing will be used. Get your key from ');
|
||||||
const a = el.createEl('a', {
|
const a = el.createEl('a', {
|
||||||
|
|||||||
@ -50,7 +50,7 @@ const CustomProviderSettings: React.FC<CustomProviderSettingsProps> = ({ plugin,
|
|||||||
|
|
||||||
const handleSettingsUpdate = async (newSettings: InfioSettings) => {
|
const handleSettingsUpdate = async (newSettings: InfioSettings) => {
|
||||||
await plugin.setSettings(newSettings);
|
await plugin.setSettings(newSettings);
|
||||||
// 使用父组件传入的回调函数来刷新整个容器
|
// Use the callback function passed from the parent component to refresh the entire container
|
||||||
onSettingsUpdate?.();
|
onSettingsUpdate?.();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -237,6 +237,7 @@ export const InfioSettingsSchema = z.object({
|
|||||||
|
|
||||||
// Files Search
|
// Files Search
|
||||||
filesSearchMethod: z.enum(['regex', 'semantic', 'auto']).catch('auto'),
|
filesSearchMethod: z.enum(['regex', 'semantic', 'auto']).catch('auto'),
|
||||||
|
ripgrepPath: z.string().catch(''),
|
||||||
|
|
||||||
/// [compatible]
|
/// [compatible]
|
||||||
// activeModels [compatible]
|
// activeModels [compatible]
|
||||||
|
|||||||
@ -467,7 +467,14 @@ export class PromptGenerator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async getSystemMessageNew(mode: Mode, filesSearchMethod: string, preferredLanguage: string): Promise<RequestMessage> {
|
private async getSystemMessageNew(mode: Mode, filesSearchMethod: string, preferredLanguage: string): Promise<RequestMessage> {
|
||||||
const systemPrompt = await SYSTEM_PROMPT(this.app.vault.getRoot().path, false, mode, filesSearchMethod, preferredLanguage, this.diffStrategy)
|
const systemPrompt = await SYSTEM_PROMPT(
|
||||||
|
this.app.vault.getRoot().path,
|
||||||
|
false,
|
||||||
|
mode,
|
||||||
|
filesSearchMethod,
|
||||||
|
preferredLanguage,
|
||||||
|
this.diffStrategy
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
role: 'system',
|
role: 'system',
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user