删除 pnpm-lock.yaml 文件,更新嵌入管理器以优化消息处理和请求管理逻辑,调整模型加载和嵌入过程中的类型定义,确保更好的稳定性和性能。

This commit is contained in:
duanfuxiang 2025-07-05 12:14:24 +08:00
parent c657a50563
commit 4b7efe8d29
3 changed files with 17 additions and 15197 deletions

15168
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -21,9 +21,21 @@ export interface TokenCountResult {
tokens: number;
}
// Worker 消息类型定义
interface WorkerMessage {
id: number;
result?: unknown;
error?: string;
}
interface WorkerRequest {
resolve: (value: unknown) => void;
reject: (reason?: unknown) => void;
}
export class EmbeddingManager {
private worker: Worker;
private requests = new Map<number, { resolve: (value: any) => void; reject: (reason?: any) => void }>();
private requests = new Map<number, WorkerRequest>();
private nextRequestId = 0;
private isModelLoaded = false;
private currentModelId: string | null = null;
@ -35,7 +47,7 @@ export class EmbeddingManager {
// 统一监听来自 Worker 的所有消息
this.worker.onmessage = (event) => {
try {
const { id, result, error } = event.data;
const { id, result, error } = event.data as WorkerMessage;
// 根据返回的 id 找到对应的 Promise 回调
const request = this.requests.get(id);
@ -53,7 +65,7 @@ export class EmbeddingManager {
console.error("Error processing worker message:", err);
// 拒绝所有待处理的请求
this.requests.forEach(request => {
request.reject(new Error(`Worker message processing error: ${err.message}`));
request.reject(new Error(`Worker message processing error: ${(err as Error).message}`));
});
this.requests.clear();
}
@ -73,14 +85,7 @@ export class EmbeddingManager {
};
}
/**
* Worker Promise Promise
* @param method (e.g., 'load', 'embed_batch')
* @param params
*/
private postRequest<T>(method: string, params: any): Promise<T> {
private postRequest<T>(method: string, params: unknown): Promise<T> {
return new Promise<T>((resolve, reject) => {
const id = this.nextRequestId++;
this.requests.set(id, { resolve, reject });
@ -88,11 +93,6 @@ export class EmbeddingManager {
});
}
/**
* Worker
* @param modelId ID, 'TaylorAI/bge-micro-v2'
* @param useGpu 使GPU加速false
*/
public async loadModel(modelId: string, useGpu: boolean = false): Promise<ModelLoadResult> {
console.log(`Loading embedding model: ${modelId}, GPU: ${useGpu}`);

View File

@ -1,7 +1,5 @@
// 完整的嵌入 Worker使用 Transformers.js
console.log('Embedding worker loaded');
// 类型定义
interface EmbedInput {
embed_input: string;
}
@ -55,7 +53,6 @@ async function loadTransformers() {
env.useFS = false;
env.useBrowserCache = true;
// 存储导入的函数
(globalThis as any).pipelineFactory = pipelineFactory;
(globalThis as any).AutoTokenizer = AutoTokenizer;
(globalThis as any).env = env;
@ -68,7 +65,6 @@ async function loadTransformers() {
}
}
// 加载模型
async function loadModel(modelKey: string, useGpu: boolean = false) {
try {
console.log(`Loading model: ${modelKey}, GPU: ${useGpu}`);
@ -141,7 +137,6 @@ async function loadModel(modelKey: string, useGpu: boolean = false) {
}
}
// 卸载模型
async function unloadModel() {
try {
console.log('Unloading model...');
@ -168,7 +163,6 @@ async function unloadModel() {
}
}
// 计算 token 数量
async function countTokens(input: string) {
try {
if (!tokenizer) {
@ -184,7 +178,6 @@ async function countTokens(input: string) {
}
}
// 生成嵌入向量
async function embedBatch(inputs: EmbedInput[]): Promise<EmbedResult[]> {
try {
if (!pipeline || !tokenizer) {
@ -201,7 +194,7 @@ async function embedBatch(inputs: EmbedInput[]): Promise<EmbedResult[]> {
}
// 批处理大小(可以根据需要调整)
const batchSize = 8;
const batchSize = 1;
if (filteredInputs.length > batchSize) {
console.log(`Processing ${filteredInputs.length} inputs in batches of ${batchSize}`);
@ -224,7 +217,6 @@ async function embedBatch(inputs: EmbedInput[]): Promise<EmbedResult[]> {
}
}
// 处理单个批次
async function processBatch(batchInputs: EmbedInput[]): Promise<EmbedResult[]> {
try {
// 计算每个输入的 token 数量
@ -295,7 +287,6 @@ async function processBatch(batchInputs: EmbedInput[]): Promise<EmbedResult[]> {
}
}
// 处理消息
async function processMessage(data: WorkerMessage): Promise<WorkerResponse> {
const { method, params, id, worker_id } = data;
@ -362,7 +353,6 @@ async function processMessage(data: WorkerMessage): Promise<WorkerResponse> {
}
}
// 监听消息
self.addEventListener('message', async (event) => {
try {
console.log('Worker received message:', event.data);
@ -389,7 +379,6 @@ self.addEventListener('message', async (event) => {
}
});
// 添加全局错误处理
self.addEventListener('error', (event) => {
console.error('Worker global error:', event);
self.postMessage({
@ -398,7 +387,6 @@ self.addEventListener('error', (event) => {
});
});
// 添加未处理的 Promise 拒绝处理
self.addEventListener('unhandledrejection', (event) => {
console.error('Worker unhandled promise rejection:', event);
self.postMessage({