2bf1fce32a
* perf: text encoding * perf: leave team code * perf: full text search code * fix: http status * perf: embedding search and vector avatar
41 lines
777 B
TypeScript
41 lines
777 B
TypeScript
import iconv from 'iconv-lite';
|
|
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
|
|
|
|
const rawEncodingList = [
|
|
'ascii',
|
|
'utf8',
|
|
'utf-8',
|
|
'utf16le',
|
|
'utf-16le',
|
|
'ucs2',
|
|
'ucs-2',
|
|
'base64',
|
|
'base64url',
|
|
'latin1',
|
|
'binary',
|
|
'hex'
|
|
];
|
|
|
|
// 加载源文件内容
|
|
export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
|
|
const content = (() => {
|
|
try {
|
|
if (rawEncodingList.includes(encoding)) {
|
|
return buffer.toString(encoding as BufferEncoding);
|
|
}
|
|
|
|
if (encoding) {
|
|
return iconv.decode(buffer, encoding);
|
|
}
|
|
|
|
return buffer.toString('utf-8');
|
|
} catch (error) {
|
|
return buffer.toString('utf-8');
|
|
}
|
|
})();
|
|
|
|
return {
|
|
rawText: content
|
|
};
|
|
};
|