dc1119ca90
* feat: toolNode aiNode readFileNode adapt new version * update docker-compose * update tip * feat: adapt new file version * perf: file input * fix: ts
26 lines
709 B
TypeScript
26 lines
709 B
TypeScript
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
|
|
import { countPromptTokens } from '../../common/string/tiktoken/index';
|
|
|
|
/* filter search result */
|
|
export const filterSearchResultsByMaxChars = async (
|
|
list: SearchDataResponseItemType[],
|
|
maxTokens: number
|
|
) => {
|
|
const results: SearchDataResponseItemType[] = [];
|
|
let totalTokens = 0;
|
|
|
|
for (let i = 0; i < list.length; i++) {
|
|
const item = list[i];
|
|
totalTokens += await countPromptTokens(item.q + item.a);
|
|
if (totalTokens > maxTokens + 500) {
|
|
break;
|
|
}
|
|
results.push(item);
|
|
if (totalTokens > maxTokens) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return results.length === 0 ? list.slice(0, 1) : results;
|
|
};
|