RYDE-WORK/Langchain-Chatchat
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/Langchain-Chatchat.git synced 2026-01-19 13:23:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

删除掉一些没用的注释，已经不需要todo了

Browse Source
This commit is contained in:
zR 2024-01-22 13:42:52 +08:00
parent 1adff316e5
commit 56d32a9908
24 changed files with 21 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
configs/server_config.py.example
View File

@ -132,12 +132,6 @@ FSCHAT_MODEL_WORKERS = {
},
}
# fastchat multi model worker server
FSCHAT_MULTI_MODEL_WORKERS = {
# TODO:
}
# fastchat controller server
FSCHAT_CONTROLLER = {
"host": DEFAULT_BIND_HOST,
"port": 20001,

3
document_loaders/mypdfloader.py
View File

@ -16,11 +16,8 @@ class RapidOCRPDFLoader(UnstructuredFileLoader):
b_unit = tqdm.tqdm(total=doc.page_count, desc="RapidOCRPDFLoader context page index: 0")
for i, page in enumerate(doc):
# 更新描述
b_unit.set_description("RapidOCRPDFLoader context page index: {}".format(i))
# 立即显示进度条更新结果
b_unit.refresh()
# TODO: 依据文本与图片顺序调整处理方式
text = page.get_text("")
resp += text + "\n"

1
server/embeddings_api.py
View File

@ -16,7 +16,6 @@ def embed_texts(
) -> BaseResponse:
'''
对文本进行向量化。返回数据格式：BaseResponse(data=List[List[float]])
TODO: 也许需要加入缓存机制，减少 token 消耗
'''
try:
if embed_model in list_embed_models(): # 使用本地Embeddings模型

2
server/knowledge_base/kb_cache/faiss_cache.py
View File

@ -55,8 +55,6 @@ class _FaissPool(CachePool):
embed_model: str = EMBEDDING_MODEL,
embed_device: str = embedding_device(),
) -> FAISS:
# TODO: 整个Embeddings加载逻辑有些混乱，待清理
# create an empty vector store
embeddings = EmbeddingsFunAdapter(embed_model)
doc = Document(page_content="init", metadata={})
vector_store = FAISS.from_documents([doc], embeddings, normalize_L2=True,distance_strategy="METRIC_INNER_PRODUCT")

2
server/knowledge_base/kb_doc_api.py
View File

@ -95,7 +95,6 @@ def _save_files_in_thread(files: List[UploadFile],
and not override
and os.path.getsize(file_path) == len(file_content)
):
# TODO: filesize 不同后的处理
file_status = f"文件 {filename} 已存在。"
logger.warn(file_status)
return dict(code=404, msg=file_status, data=data)
@ -116,7 +115,6 @@ def _save_files_in_thread(files: List[UploadFile],
yield result
# TODO: 等langchain.document_loaders支持内存文件的时候再开通
# def files2docs(files: List[UploadFile] = File(..., description="上传文件，支持多文件"),
# knowledge_base_name: str = Form(..., description="知识库名称", examples=["samples"]),
# override: bool = Form(False, description="覆盖已有文件"),

1
server/knowledge_base/kb_service/base.py
View File

@ -191,7 +191,6 @@ class KBService(ABC):
'''
传入参数为： {doc_id: Document, ...}
如果对应 doc_id 的值为 None，或其 page_content 为空，则删除该文档
TODO：是否要支持新增 docs ？
'''
self.del_doc_by_ids(list(docs.keys()))
docs = []

1
server/knowledge_base/kb_service/milvus_kb_service.py
View File

@ -70,7 +70,6 @@ class MilvusKBService(KBService):
return score_threshold_process(score_threshold, top_k, docs)
def do_add_doc(self, docs: List[Document], **kwargs) -> List[Dict]:
# TODO: workaround for bug #10492 in langchain
for doc in docs:
for k, v in doc.metadata.items():
doc.metadata[k] = str(v)

2
server/knowledge_base/kb_service/pg_kb_service.py
View File

@ -32,8 +32,6 @@ class PGKBService(KBService):
results = [Document(page_content=row[0], metadata=row[1]) for row in
session.execute(stmt, {'ids': ids}).fetchall()]
return results
# TODO:
def del_doc_by_ids(self, ids: List[str]) -> bool:
return super().del_doc_by_ids(ids)

1
server/knowledge_base/kb_summary/base.py
View File

@ -13,7 +13,6 @@ from server.db.repository.knowledge_metadata_repository import add_summary_to_db
from langchain.docstore.document import Document
# TODO 暂不考虑文件更新，需要重新删除相关文档，再重新添加
class KBSummaryService(ABC):
kb_name: str
embed_model: str

6
server/knowledge_base/kb_summary/summary_chunk.py
View File

@ -112,12 +112,6 @@ class SummaryAdapter:
docs: List[DocumentWithVSId] = []) -> List[Document]:
logger.info("start summary")
# TODO 暂不处理文档中涉及语义重复、上下文缺失、document was longer than the context length 的问题
# merge_docs = self._drop_overlap(docs)
# # 将merge_docs中的句子合并成一个文档
# text = self._join_docs(merge_docs)
# 根据段落于句子的分隔符，将文档分成chunk，每个chunk长度小于token_max长度
"""
这个过程分成两个部分：
1. 对每个文档进行处理，得到每个文档的摘要

1
server/knowledge_base/utils.py
View File

@ -174,7 +174,6 @@ def get_loader(loader_name: str, file_path: str, loader_kwargs: Dict = None):
if encode_detect is None:
encode_detect = {"encoding": "utf-8"}
loader_kwargs["encoding"] = encode_detect["encoding"]
## TODO：支持更多的自定义CSV读取逻辑
elif loader_name == "JSONLoader":
loader_kwargs.setdefault("jq_schema", ".")

2
server/model_workers/azure.py
View File

@ -67,12 +67,10 @@ class AzureWorker(ApiModelWorker):
self.logger.error(f"请求 Azure API 时发生错误：{resp}")
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="You are a helpful, respectful and honest assistant.",

2
server/model_workers/baichuan.py
View File

@ -88,12 +88,10 @@ class BaiChuanWorker(ApiModelWorker):
yield data
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="",

2
server/model_workers/base.py
View File

@ -125,8 +125,6 @@ class ApiModelWorker(BaseModelWorker):
def count_token(self, params):
# TODO：需要完善
# print("count token")
prompt = params["prompt"]
return {"count": len(str(prompt)), "error_code": 0}

3
server/model_workers/fangzhou.py
View File

@ -21,7 +21,7 @@ class FangZhouWorker(ApiModelWorker):
**kwargs,
):
kwargs.update(model_names=model_names, controller_addr=controller_addr, worker_addr=worker_addr)
kwargs.setdefault("context_len", 16384) # TODO: 不同的模型有不同的大小
kwargs.setdefault("context_len", 16384)
super().__init__(**kwargs)
self.version = version
@ -77,7 +77,6 @@ class FangZhouWorker(ApiModelWorker):
break
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)

1
server/model_workers/gemini.py
View File

@ -95,7 +95,6 @@ class GeminiWorker(ApiModelWorker):
print("Invalid JSON string:", json_string)
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)

5
server/model_workers/minimax.py
View File

@ -37,7 +37,6 @@ class MiniMaxWorker(ApiModelWorker):
def do_chat(self, params: ApiChatParams) -> Dict:
# 按照官网推荐，直接调用abab 5.5模型
# TODO: 支持指定回复要求，支持指定用户名称、AI名称
params.load_config(self.model_names[0])
url = 'https://api.minimax.chat/v1/text/chatcompletion{pro}?GroupId={group_id}'
@ -55,7 +54,7 @@ class MiniMaxWorker(ApiModelWorker):
"temperature": params.temperature,
"top_p": params.top_p,
"tokens_to_generate": params.max_tokens or 1024,
# TODO: 以下参数为minimax特有，传入空值会出错。
# 以下参数为minimax特有，传入空值会出错。
# "prompt": params.system_message or self.conv.system_message,
# "bot_setting": [],
# "role_meta": params.role_meta,
@ -143,12 +142,10 @@ class MiniMaxWorker(ApiModelWorker):
return {"code": 200, "data": result}
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="你是MiniMax自主研发的大型语言模型，回答问题简洁有条理。",

3
server/model_workers/qianfan.py
View File

@ -187,14 +187,11 @@ class QianFanWorker(ApiModelWorker):
i += batch_size
return {"code": 200, "data": result}
# TODO: qianfan支持续写模型
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="你是一个聪明的助手，请根据用户的提示来完成任务",

2
server/model_workers/qwen.py
View File

@ -100,12 +100,10 @@ class QwenWorker(ApiModelWorker):
return {"code": 200, "data": result}
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="你是一个聪明、对人类有帮助的人工智能，你可以对人类提出的问题给出有用、详细、礼貌的回答。",

2
server/model_workers/tiangong.py
View File

@ -70,12 +70,10 @@ class TianGongWorker(ApiModelWorker):
yield data
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="",

3
server/model_workers/xinghuo.py
View File

@ -42,7 +42,6 @@ class XingHuoWorker(ApiModelWorker):
self.version = version
def do_chat(self, params: ApiChatParams) -> Dict:
# TODO: 当前每次对话都要重新连接websocket，确认是否可以保持连接
params.load_config(self.model_names[0])
version_mapping = {
@ -73,12 +72,10 @@ class XingHuoWorker(ApiModelWorker):
yield {"error_code": 0, "text": text}
def get_embeddings(self, params):
# TODO: 支持embeddings
print("embedding")
print(params)
def make_conv_template(self, conv_template: str = None, model_path: str = None) -> Conversation:
# TODO: 确认模板是否需要修改
return conv.Conversation(
name=self.model_names[0],
system_message="你是一个聪明的助手，请根据用户的提示来完成任务",

7
server/utils.py
View File

@ -36,7 +36,6 @@ async def wrap_done(fn: Awaitable, event: asyncio.Event):
await fn
except Exception as e:
logging.exception(e)
# TODO: handle exception
msg = f"Caught exception: {e}"
logger.error(f'{e.__class__.__name__}: {msg}',
exc_info=e if log_verbose else None)
@ -404,7 +403,7 @@ def fschat_controller_address() -> str:
def fschat_model_worker_address(model_name: str = LLM_MODELS[0]) -> str:
if model := get_model_worker_config(model_name): # TODO: depends fastchat
if model := get_model_worker_config(model_name):
host = model["host"]
if host == "0.0.0.0":
host = "127.0.0.1"
@ -449,7 +448,7 @@ def get_prompt_template(type: str, name: str) -> Optional[str]:
from configs import prompt_config
import importlib
importlib.reload(prompt_config) # TODO: 检查configs/prompt_config.py文件有修改再重新加载
importlib.reload(prompt_config)
return prompt_config.PROMPT_TEMPLATES[type].get(name)
@ -550,7 +549,7 @@ def run_in_thread_pool(
thread = pool.submit(func, **kwargs)
tasks.append(thread)
for obj in as_completed(tasks): # TODO: Ctrl+c无法停止
for obj in as_completed(tasks):
yield obj.result()

2
startup.py
View File

@ -418,7 +418,7 @@ def run_openai_api(log_level: str = "INFO", started_event: mp.Event = None):
set_httpx_config()
controller_addr = fschat_controller_address()
app = create_openai_api_app(controller_addr, log_level=log_level) # TODO: not support keys yet.
app = create_openai_api_app(controller_addr, log_level=log_level)
_set_app_event(app, started_event)
host = FSCHAT_OPENAI_API["host"]

1
webui_pages/dialogue/dialogue.py
View File

@ -126,7 +126,6 @@ def dialogue_page(api: ApiRequest, is_lite: bool = False):
chat_box.use_chat_name(conversation_name)
conversation_id = st.session_state["conversation_ids"][conversation_name]
# TODO: 对话模型与会话绑定
def on_mode_change():
mode = st.session_state.dialogue_mode
text = f"已切换到 {mode} 模式。"