RYDE-WORK/Langchain-Chatchat
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/Langchain-Chatchat.git synced 2026-02-03 21:23:13 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

修复csv文件解析编码报错的问题。 (#1508)

Browse Source
This commit is contained in:
peterz3g 2023-09-21 14:41:49 +08:00 committed by GitHub
parent 6a0a791c80
commit 3ff130ecc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
server/knowledge_base/utils.py
View File

@ -26,6 +26,7 @@ from concurrent.futures import ThreadPoolExecutor
from server.utils import run_in_thread_pool, embedding_device from server.utils import run_in_thread_pool, embedding_device
import io import io
from typing import List, Union, Callable, Dict, Optional, Tuple, Generator from typing import List, Union, Callable, Dict, Optional, Tuple, Generator
import chardet
def validate_kb_name(knowledge_base_id: str) -> bool: def validate_kb_name(knowledge_base_id: str) -> bool:
@ -167,7 +168,14 @@ def get_loader(loader_name: str, file_path_or_content: Union[str, bytes, io.Stri
if loader_name == "UnstructuredFileLoader": if loader_name == "UnstructuredFileLoader":
loader = DocumentLoader(file_path_or_content, autodetect_encoding=True) loader = DocumentLoader(file_path_or_content, autodetect_encoding=True)
elif loader_name == "CSVLoader": elif loader_name == "CSVLoader":
loader = DocumentLoader(file_path_or_content, encoding="utf-8") # 自动识别文件编码类型，避免langchain loader 加载文件报编码错误
with open(file_path_or_content, 'rb') as struct_file:
encode_detect = chardet.detect(struct_file.read())
if encode_detect:
loader = DocumentLoader(file_path_or_content, encoding=encode_detect["encoding"])
else:
loader = DocumentLoader(file_path_or_content, encoding="utf-8")
elif loader_name == "JSONLoader": elif loader_name == "JSONLoader":
loader = DocumentLoader(file_path_or_content, jq_schema=".", text_content=False) loader = DocumentLoader(file_path_or_content, jq_schema=".", text_content=False)
elif loader_name == "CustomJSONLoader": elif loader_name == "CustomJSONLoader":