From f21e00bf16712dd6f5c3fe41eeb585747ad3b2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=83=BD=E5=88=9A=E5=93=A5?= Date: Sun, 18 Jun 2023 17:30:41 +0800 Subject: [PATCH 01/14] =?UTF-8?q?LLM=E5=AF=B9=E8=AF=9D=E5=92=8C=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E5=AF=B9=E8=AF=9D=E6=8E=A5=E5=8F=A3=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=B5=81=E5=BC=8F=E8=BE=93=E5=87=BA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 81 ++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/api.py b/api.py index 0413c95b..89afe864 100644 --- a/api.py +++ b/api.py @@ -8,11 +8,13 @@ import urllib import nltk import pydantic import uvicorn -from fastapi import Body, FastAPI, File, Form, Query, UploadFile, WebSocket +from fastapi import Body, Request, FastAPI, File, Form, Query, UploadFile, WebSocket from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse from pydantic import BaseModel from typing_extensions import Annotated from starlette.responses import RedirectResponse +from sse_starlette.sse import EventSourceResponse from chains.local_doc_qa import LocalDocQA from configs.model_config import (KB_ROOT_PATH, EMBEDDING_DEVICE, @@ -266,6 +268,7 @@ async def update_doc( async def local_doc_chat( knowledge_base_id: str = Body(..., description="Knowledge Base Name", example="kb1"), question: str = Body(..., description="Question", example="工伤保险是什么？"), + stream: bool = Body(False, description="是否开启流式输出，默认false，有些模型可能不支持。"), history: List[List[str]] = Body( [], description="History of previous questions and answers", @@ -287,22 +290,34 @@ async def local_doc_chat( source_documents=[], ) else: - for resp, history in local_doc_qa.get_knowledge_based_answer( - query=question, vs_path=vs_path, chat_history=history, streaming=True - ): - pass - source_documents = [ - f"""出处 [{inum + 1}] {os.path.split(doc.metadata['source'])[-1]}：\n\n{doc.page_content}\n\n""" - f"""相关度：{doc.metadata['score']}\n\n""" - for inum, doc in enumerate(resp["source_documents"]) - ] + if (stream): + def generate_answer (): + last_print_len = 0 + for resp, next_history in local_doc_qa.get_knowledge_based_answer( + query=question, vs_path=vs_path, chat_history=history, streaming=True + ): + yield resp["result"][last_print_len:] + last_print_len=len(resp["result"]) - return ChatMessage( - question=question, - response=resp["result"], - history=history, - source_documents=source_documents, - ) + return StreamingResponse(generate_answer()) + else: + for resp, next_history in local_doc_qa.get_knowledge_based_answer( + query=question, vs_path=vs_path, chat_history=history, streaming=True + ): + pass + + source_documents = [ + f"""出处 [{inum + 1}] {os.path.split(doc.metadata['source'])[-1]}：\n\n{doc.page_content}\n\n""" + f"""相关度：{doc.metadata['score']}\n\n""" + for inum, doc in enumerate(resp["source_documents"]) + ] + + return ChatMessage( + question=question, + response=resp["result"], + history=next_history, + source_documents=source_documents, + ) async def bing_search_chat( @@ -337,6 +352,7 @@ async def bing_search_chat( async def chat( question: str = Body(..., description="Question", example="工伤保险是什么？"), + stream: bool = Body(False, description="是否开启流式输出，默认false，有些模型可能不支持。"), history: List[List[str]] = Body( [], description="History of previous questions and answers", @@ -348,18 +364,29 @@ async def chat( ], ), ): - for answer_result in local_doc_qa.llm.generatorAnswer(prompt=question, history=history, - streaming=True): - resp = answer_result.llm_output["answer"] - history = answer_result.history - pass - return ChatMessage( - question=question, - response=resp, - history=history, - source_documents=[], - ) + if (stream): + def generate_answer (): + last_print_len = 0 + for answer_result in local_doc_qa.llm.generatorAnswer(prompt=question, history=history, + streaming=True): + yield answer_result.llm_output["answer"][last_print_len:] + last_print_len = len(answer_result.llm_output["answer"]) + + return StreamingResponse(generate_answer()) + else: + for answer_result in local_doc_qa.llm.generatorAnswer(prompt=question, history=history, + streaming=True): + resp = answer_result.llm_output["answer"] + history = answer_result.history + pass + + return ChatMessage( + question=question, + response=resp, + history=history, + source_documents=[], + ) async def stream_chat(websocket: WebSocket, knowledge_base_id: str): From d61ff37f45db4da4078600827514b81bdf50e2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=83=BD=E5=88=9A=E5=93=A5?= Date: Sun, 18 Jun 2023 17:35:55 +0800 Subject: [PATCH 02/14] =?UTF-8?q?=E5=A2=9E=E5=8A=A0sse=5Fstarlette?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b067c19e..39b61f4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ cpm_kernels faiss-cpu gradio==3.28.3 fastapi~=0.95.0 +sse_starlette uvicorn~=0.21.1 pypinyin~=0.48.0 click~=8.1.3 @@ -34,4 +35,4 @@ tqdm~=4.65.0 requests~=2.28.2 tenacity~=8.2.2 # 默认下载的charset_normalizer模块版本过高会抛出，`artially initialized module 'charset_normalizer' has no attribute 'md__mypyc' (most likely due to a circular import)` -charset_normalizer==2.1.0 \ No newline at end of file +charset_normalizer==2.1.0 From 6023426344d6f3b7e27abf20a68497061863f8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=83=BD=E5=88=9A=E5=93=A5?= Date: Sun, 18 Jun 2023 17:36:46 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39b61f4d..a22f0ae0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ cpm_kernels faiss-cpu gradio==3.28.3 fastapi~=0.95.0 -sse_starlette uvicorn~=0.21.1 pypinyin~=0.48.0 click~=8.1.3 From 41d8846e136c7536016ff0c01183239b5a53822a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=83=BD=E5=88=9A=E5=93=A5?= Date: Sun, 18 Jun 2023 17:37:26 +0800 Subject: [PATCH 04/14] =?UTF-8?q?=E7=A7=BB=E9=99=A4EventSourceResponse?= =?UTF-8?q?=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api.py b/api.py index 89afe864..768a3f73 100644 --- a/api.py +++ b/api.py @@ -14,7 +14,6 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel from typing_extensions import Annotated from starlette.responses import RedirectResponse -from sse_starlette.sse import EventSourceResponse from chains.local_doc_qa import LocalDocQA from configs.model_config import (KB_ROOT_PATH, EMBEDDING_DEVICE, From f21fea41b4e4f5bda40e2caca0c8c3cac0f906e9 Mon Sep 17 00:00:00 2001 From: liunux4odoo Date: Tue, 25 Jul 2023 08:53:28 +0800 Subject: [PATCH 05/14] =?UTF-8?q?bug=20fix:=20api.py=E4=B8=AD,=E6=89=80?= =?UTF-8?q?=E6=9C=89chat=E6=8E=A5=E5=8F=A3=E7=9A=84=E5=8F=82=E6=95=B0histo?= =?UTF-8?q?ry,=E5=85=B6=E5=85=83=E7=B4=A0=E5=BA=94=E6=94=AF=E6=8C=81None?= =?UTF-8?q?=EF=BC=8C=E6=96=B9=E4=BE=BF=E5=85=B6=E4=BB=96=E6=A1=86=E6=9E=B6?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E3=80=82=20=E5=90=A6=E5=88=99=E4=BB=A5histor?= =?UTF-8?q?y=3D[[None,'some=20thing']]=E8=B0=83=E7=94=A8=E6=97=B6=E4=BC=9A?= =?UTF-8?q?=E8=BF=94=E5=9B=9E422=E9=94=99=E8=AF=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api.py b/api.py index 5c348a95..a8b29d81 100644 --- a/api.py +++ b/api.py @@ -55,7 +55,7 @@ class ListDocsResponse(BaseResponse): class ChatMessage(BaseModel): question: str = pydantic.Field(..., description="Question text") response: str = pydantic.Field(..., description="Response text") - history: List[List[str]] = pydantic.Field(..., description="History text") + history: List[List[Optional[str]]] = pydantic.Field(..., description="History text") source_documents: List[str] = pydantic.Field( ..., description="List of source documents and their scores" ) @@ -303,7 +303,7 @@ async def update_doc( async def local_doc_chat( knowledge_base_id: str = Body(..., description="Knowledge Base Name", example="kb1"), question: str = Body(..., description="Question", example="工伤保险是什么？"), - history: List[List[str]] = Body( + history: List[List[Optional[str]]] = Body( [], description="History of previous questions and answers", example=[ @@ -344,7 +344,7 @@ async def local_doc_chat( async def bing_search_chat( question: str = Body(..., description="Question", example="工伤保险是什么？"), - history: Optional[List[List[str]]] = Body( + history: Optional[List[List[Optional[str]]]] = Body( [], description="History of previous questions and answers", example=[ @@ -374,7 +374,7 @@ async def bing_search_chat( async def chat( question: str = Body(..., description="Question", example="工伤保险是什么？"), - history: Optional[List[List[str]]] = Body( + history: Optional[List[List[Optional[str]]]] = Body( [], description="History of previous questions and answers", example=[ From 466f0c9c97658e9bbe8fe7ad72b76c7a20f6de05 Mon Sep 17 00:00:00 2001 From: YuhaoWU <31399290+KennyNg-19@users.noreply.github.com> Date: Tue, 25 Jul 2023 16:58:25 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E5=AE=89=E8=A3=85=E6=95=99=E7=A8=8Bconda?= =?UTF-8?q?=20create=E6=96=B0=E5=A2=9E=E9=BB=98=E8=AE=A4=E7=9A=84/envs?= =?UTF-8?q?=E4=B8=8B=E9=9D=A2=EF=BC=8C=E8=80=8C=E4=B8=8D=E5=8F=AA=E6=98=AF?= =?UTF-8?q?=E5=8F=AA=E8=83=BD=E8=A6=81=E9=A2=9D=E5=A4=96=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=20=E7=94=A8-p=20=20(#482)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update 安装教程中conda虚拟环境的创建与激活 --- README_en.md | 4 ++++ docs/INSTALL.md | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/README_en.md b/README_en.md index 79edb913..249962da 100644 --- a/README_en.md +++ b/README_en.md @@ -87,6 +87,10 @@ $ conda create -p /your_path/env_name python=3.8 # Activate the environment $ source activate /your_path/env_name +# or, do not specify an env path, note that /your_path/env_name is to be replaced with env_name below +$ conda create -n env_name python=3.8 +$ conda activate env_name # Activate the environment + # Deactivate the environment $ source deactivate /your_path/env_name diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 2682c7b7..cd3c51d4 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -12,6 +12,12 @@ $ conda create -p /your_path/env_name python=3.8 # 激活环境 $ source activate /your_path/env_name + +# 或，conda安装，不指定路径, 注意以下，都将/your_path/env_name替换为env_name +$ conda create -n env_name python=3.8 +$ conda activate env_name # Activate the environment + +# 更新py库 $ pip3 install --upgrade pip # 关闭环境 From 5f74f70515ef687d678e070415e0f787955cf532 Mon Sep 17 00:00:00 2001 From: liunux4odoo Date: Tue, 25 Jul 2023 22:36:49 +0800 Subject: [PATCH 07/14] =?UTF-8?q?fix:=20streamlit=20ui=E5=9C=A8=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=A4=9A=E4=B8=AA=E6=96=87=E4=BB=B6=E5=88=B0=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E6=97=B6=E5=87=BA=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- webui_st.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/webui_st.py b/webui_st.py index 1584a55a..83092104 100644 --- a/webui_st.py +++ b/webui_st.py @@ -1,6 +1,8 @@ import streamlit as st from streamlit_chatbox import st_chatbox import tempfile +from pathlib import Path + ###### 从webui借用的代码 ##### ###### 做了少量修改 ##### import os @@ -101,23 +103,23 @@ def get_answer(query, vs_path, history, mode, score_threshold=VECTOR_SEARCH_SCOR def get_vector_store(vs_id, files, sentence_size, history, one_conent, one_content_segmentation): - vs_path = os.path.join(KB_ROOT_PATH, vs_id, "vector_store") - filelist = [] - if not os.path.exists(os.path.join(KB_ROOT_PATH, vs_id, "content")): - os.makedirs(os.path.join(KB_ROOT_PATH, vs_id, "content")) + vs_path = Path(KB_ROOT_PATH) / vs_id / "vector_store" + con_path = Path(KB_ROOT_PATH) / vs_id / "content" + con_path.mkdir(parents=True, exist_ok=True) + qa = st.session_state.local_doc_qa if qa.llm_model_chain and qa.embeddings: + filelist = [] if isinstance(files, list): for file in files: filename = os.path.split(file.name)[-1] - shutil.move(file.name, os.path.join( - KB_ROOT_PATH, vs_id, "content", filename)) - filelist.append(os.path.join( - KB_ROOT_PATH, vs_id, "content", filename)) + target = con_path / filename + shutil.move(file.name, target) + filelist.append(str(target)) vs_path, loaded_files = qa.init_knowledge_vector_store( - filelist, vs_path, sentence_size) + filelist, str(vs_path), sentence_size) else: - vs_path, loaded_files = qa.one_knowledge_add(vs_path, files, one_conent, one_content_segmentation, + vs_path, loaded_files = qa.one_knowledge_add(str(vs_path), files, one_conent, one_content_segmentation, sentence_size) if len(loaded_files): file_status = f"已添加 {'、'.join([os.path.split(i)[-1] for i in loaded_files if i])} 内容至知识库，并已加载知识库，请开始提问" @@ -322,7 +324,8 @@ with st.sidebar: sentence_size = st.slider('文本入库分句长度限制', 1, 1000, SENTENCE_SIZE) files = st.file_uploader('上传知识文件', ['docx', 'txt', 'md', 'csv', 'xlsx', 'pdf'], - accept_multiple_files=True) + accept_multiple_files=True, + ) if st.button('添加文件到知识库'): temp_dir = tempfile.mkdtemp() file_list = [] @@ -331,8 +334,8 @@ with st.sidebar: with open(file, 'wb') as fp: fp.write(f.getvalue()) file_list.append(TempFile(file)) - _, _, history = get_vector_store( - vs_path, file_list, sentence_size, [], None, None) + _, _, history = get_vector_store( + vs_path, file_list, sentence_size, [], None, None) st.session_state.files = [] From 58a5de92a5a7453a11895a32163a8474e5820732 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Wed, 26 Jul 2023 17:05:37 +0800 Subject: [PATCH 08/14] =?UTF-8?q?1.=E6=9B=B4=E6=94=B9=E5=8A=A0=E8=BD=BDlor?= =?UTF-8?q?a=E7=9A=84=E6=96=B9=E5=BC=8F;2.=E5=85=81=E8=AE=B8api.py?= =?UTF-8?q?=E8=B0=83=E7=94=A8args.py=E7=9A=84=E5=91=BD=E4=BB=A4=E8=A1=8C;3?= =?UTF-8?q?.=20FastChat=E8=B7=AF=E5=BE=84=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 2 +- configs/model_config.py | 11 ++++------- models/loader/args.py | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/api.py b/api.py index bc2fe633..452313e5 100644 --- a/api.py +++ b/api.py @@ -583,7 +583,7 @@ if __name__ == "__main__": parser.add_argument("--ssl_keyfile", type=str) parser.add_argument("--ssl_certfile", type=str) # 初始化消息 - args = None + args = parser.parse_args() args_dict = vars(args) shared.loaderCheckPoint = LoaderCheckPoint(args_dict) diff --git a/configs/model_config.py b/configs/model_config.py index 846da983..91ed20a8 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -32,7 +32,7 @@ EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backe # llm_model_dict 处理了loader的一些预设行为，如加载位置，模型名称，模型处理器实例 # 在以下字典中修改属性值，以指定本地 LLM 模型存储位置 # 如将 "chatglm-6b" 的 "local_model_path" 由 None 修改为 "User/Downloads/chatglm-6b" -# 此处请写绝对路径 +# 此处请写绝对路径,且路径中必须包含模型名称，FastChat会根据路径名称提取repo-id llm_model_dict = { "chatglm-6b-int4-qe": { "name": "chatglm-6b-int4-qe", @@ -104,17 +104,14 @@ LOAD_IN_8BIT = False # Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. BF16 = False # 本地lora存放的位置 -LORA_DIR = "loras/" - -# LLM lora path，默认为空，如果有请直接指定文件夹路径 -LLM_LORA_PATH = "" -USE_LORA = True if LLM_LORA_PATH else False +LORA_DIR = "./loras/" # LLM streaming reponse STREAMING = True # Use p-tuning-v2 PrefixEncoder -USE_PTUNING_V2 = False + +PTUNING_DIR="./ptuning-v2" # LLM running device LLM_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" diff --git a/models/loader/args.py b/models/loader/args.py index cd3e78b8..9e2e8b22 100644 --- a/models/loader/args.py +++ b/models/loader/args.py @@ -42,7 +42,7 @@ parser.add_argument('--no-remote-model', action='store_true', help='remote in th 'model to add the ` ' '--no-remote-model`') parser.add_argument('--model-name', type=str, default=LLM_MODEL, help='Name of the model to load by default.') -parser.add_argument('--lora', type=str, help='Name of the LoRA to apply to the model by default.') +parser.add_argument('--lora', type=str, action="store_true",help='Name of the LoRA to apply to the model by default.') parser.add_argument("--lora-dir", type=str, default=LORA_DIR, help="Path to directory with all the loras") parser.add_argument('--use-ptuning-v2',action='store_true',help="whether use ptuning-v2 checkpoint") parser.add_argument("--ptuning-dir",type=str,default=PTUNING_DIR,help="the dir of ptuning-v2 checkpoint") From e8ff31be1fb0ba04c158558446373b7de43c1836 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Wed, 26 Jul 2023 17:46:02 +0800 Subject: [PATCH 09/14] =?UTF-8?q?1.=E4=BD=BFapi.py=E9=87=8C=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E4=BD=BF=E7=94=A8args.py=E7=9A=84=E5=8F=82=E6=95=B0;2?= =?UTF-8?q?.=E5=85=BC=E5=AE=B9args.py=E5=92=8Cmodel=5Fconfig.py=E7=9A=84?= =?UTF-8?q?=E6=8E=A7=E5=88=B6=E6=96=B9=E5=BC=8F;3.=E6=9B=B4=E6=96=B0fastch?= =?UTF-8?q?at=E8=B0=83=E7=94=A8=E6=A8=A1=E5=9E=8B=E5=90=8D=E7=9A=84?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configs/model_config.py | 13 +++++++++---- models/loader/args.py | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/configs/model_config.py b/configs/model_config.py index 91ed20a8..74d8e186 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -32,7 +32,7 @@ EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backe # llm_model_dict 处理了loader的一些预设行为，如加载位置，模型名称，模型处理器实例 # 在以下字典中修改属性值，以指定本地 LLM 模型存储位置 # 如将 "chatglm-6b" 的 "local_model_path" 由 None 修改为 "User/Downloads/chatglm-6b" -# 此处请写绝对路径,且路径中必须包含模型名称，FastChat会根据路径名称提取repo-id +# 此处请写绝对路径 llm_model_dict = { "chatglm-6b-int4-qe": { "name": "chatglm-6b-int4-qe", @@ -104,15 +104,20 @@ LOAD_IN_8BIT = False # Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. BF16 = False # 本地lora存放的位置 -LORA_DIR = "./loras/" +LORA_DIR = "loras/" + +# LORA的名称，如有请指定为列表 + +LORA_NAME = "" +USE_LORA = True if LORA_NAME else False # LLM streaming reponse STREAMING = True # Use p-tuning-v2 PrefixEncoder +USE_PTUNING_V2 = False -PTUNING_DIR="./ptuning-v2" - +PTUNING_DIR = "./ptuning-v2" # LLM running device LLM_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" diff --git a/models/loader/args.py b/models/loader/args.py index 9e2e8b22..02f49269 100644 --- a/models/loader/args.py +++ b/models/loader/args.py @@ -42,9 +42,10 @@ parser.add_argument('--no-remote-model', action='store_true', help='remote in th 'model to add the ` ' '--no-remote-model`') parser.add_argument('--model-name', type=str, default=LLM_MODEL, help='Name of the model to load by default.') -parser.add_argument('--lora', type=str, action="store_true",help='Name of the LoRA to apply to the model by default.') +parser.add_argument("--use-lora",type=bool,default=USE_LORA,help="use lora or not") +parser.add_argument('--lora', type=str, default=LORA_NAME,help='Name of the LoRA to apply to the model by default.') parser.add_argument("--lora-dir", type=str, default=LORA_DIR, help="Path to directory with all the loras") -parser.add_argument('--use-ptuning-v2',action='store_true',help="whether use ptuning-v2 checkpoint") +parser.add_argument('--use-ptuning-v2',action=USE_PTUNING_V2,help="whether use ptuning-v2 checkpoint") parser.add_argument("--ptuning-dir",type=str,default=PTUNING_DIR,help="the dir of ptuning-v2 checkpoint") # Accelerate/transformers parser.add_argument('--load-in-8bit', action='store_true', default=LOAD_IN_8BIT, From 34a2af118bbd1f39c8f21323372c1cf4eabd1b64 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Wed, 26 Jul 2023 18:04:35 +0800 Subject: [PATCH 10/14] update model_config.py --- configs/model_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/model_config.py b/configs/model_config.py index 74d8e186..bc1a5c2c 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -32,7 +32,7 @@ EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backe # llm_model_dict 处理了loader的一些预设行为，如加载位置，模型名称，模型处理器实例 # 在以下字典中修改属性值，以指定本地 LLM 模型存储位置 # 如将 "chatglm-6b" 的 "local_model_path" 由 None 修改为 "User/Downloads/chatglm-6b" -# 此处请写绝对路径 +# 此处请写绝对路径,且路径中必须包含repo-id的模型名称，因为FastChat是以模型名匹配的 llm_model_dict = { "chatglm-6b-int4-qe": { "name": "chatglm-6b-int4-qe", From 8732dcc1c4710314332a407052988e225de386d7 Mon Sep 17 00:00:00 2001 From: liunux4odoo Date: Thu, 27 Jul 2023 08:42:17 +0800 Subject: [PATCH 11/14] =?UTF-8?q?=E6=81=A2=E5=A4=8D=E8=A2=ABPR659=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E7=9A=84model=5Fconfig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configs/model_config.py | 135 +++++++++++++++++++++++++++++++++++----- 1 file changed, 120 insertions(+), 15 deletions(-) diff --git a/configs/model_config.py b/configs/model_config.py index 846da983..ed277ee2 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -27,7 +27,6 @@ EMBEDDING_MODEL = "text2vec" # Embedding running device EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" - # supported LLM models # llm_model_dict 处理了loader的一些预设行为，如加载位置，模型名称，模型处理器实例 # 在以下字典中修改属性值，以指定本地 LLM 模型存储位置 @@ -38,44 +37,113 @@ llm_model_dict = { "name": "chatglm-6b-int4-qe", "pretrained_model_name": "THUDM/chatglm-6b-int4-qe", "local_model_path": None, - "provides": "ChatGLM" + "provides": "ChatGLMLLMChain" }, "chatglm-6b-int4": { "name": "chatglm-6b-int4", "pretrained_model_name": "THUDM/chatglm-6b-int4", "local_model_path": None, - "provides": "ChatGLM" + "provides": "ChatGLMLLMChain" }, "chatglm-6b-int8": { "name": "chatglm-6b-int8", "pretrained_model_name": "THUDM/chatglm-6b-int8", "local_model_path": None, - "provides": "ChatGLM" + "provides": "ChatGLMLLMChain" }, "chatglm-6b": { "name": "chatglm-6b", "pretrained_model_name": "THUDM/chatglm-6b", "local_model_path": None, - "provides": "ChatGLM" + "provides": "ChatGLMLLMChain" + }, + "chatglm2-6b": { + "name": "chatglm2-6b", + "pretrained_model_name": "THUDM/chatglm2-6b", + "local_model_path": None, + "provides": "ChatGLMLLMChain" + }, + "chatglm2-6b-int4": { + "name": "chatglm2-6b-int4", + "pretrained_model_name": "THUDM/chatglm2-6b-int4", + "local_model_path": None, + "provides": "ChatGLMLLMChain" + }, + "chatglm2-6b-int8": { + "name": "chatglm2-6b-int8", + "pretrained_model_name": "THUDM/chatglm2-6b-int8", + "local_model_path": None, + "provides": "ChatGLMLLMChain" }, - "chatyuan": { "name": "chatyuan", "pretrained_model_name": "ClueAI/ChatYuan-large-v2", "local_model_path": None, - "provides": None + "provides": "MOSSLLMChain" }, "moss": { "name": "moss", "pretrained_model_name": "fnlp/moss-moon-003-sft", "local_model_path": None, + "provides": "MOSSLLMChain" + }, + "moss-int4": { + "name": "moss", + "pretrained_model_name": "fnlp/moss-moon-003-sft-int4", + "local_model_path": None, "provides": "MOSSLLM" }, "vicuna-13b-hf": { "name": "vicuna-13b-hf", "pretrained_model_name": "vicuna-13b-hf", "local_model_path": None, - "provides": "LLamaLLM" + "provides": "LLamaLLMChain" + }, + "vicuna-7b-hf": { + "name": "vicuna-13b-hf", + "pretrained_model_name": "vicuna-13b-hf", + "local_model_path": None, + "provides": "LLamaLLMChain" + }, + # 直接调用返回requests.exceptions.ConnectionError错误，需要通过huggingface_hub包里的snapshot_download函数 + # 下载模型，如果snapshot_download还是返回网络错误，多试几次，一般是可以的， + # 如果仍然不行，则应该是网络加了防火墙(在服务器上这种情况比较常见)，基本只能从别的设备上下载， + # 然后转移到目标设备了. + "bloomz-7b1": { + "name": "bloomz-7b1", + "pretrained_model_name": "bigscience/bloomz-7b1", + "local_model_path": None, + "provides": "MOSSLLMChain" + + }, + # 实测加载bigscience/bloom-3b需要170秒左右，暂不清楚为什么这么慢 + # 应与它要加载专有token有关 + "bloom-3b": { + "name": "bloom-3b", + "pretrained_model_name": "bigscience/bloom-3b", + "local_model_path": None, + "provides": "MOSSLLMChain" + + }, + "baichuan-7b": { + "name": "baichuan-7b", + "pretrained_model_name": "baichuan-inc/baichuan-7B", + "local_model_path": None, + "provides": "MOSSLLMChain" + }, + # llama-cpp模型的兼容性问题参考https://github.com/abetlen/llama-cpp-python/issues/204 + "ggml-vicuna-13b-1.1-q5": { + "name": "ggml-vicuna-13b-1.1-q5", + "pretrained_model_name": "lmsys/vicuna-13b-delta-v1.1", + # 这里需要下载好模型的路径,如果下载模型是默认路径则它会下载到用户工作区的 + # /.cache/huggingface/hub/models--vicuna--ggml-vicuna-13b-1.1/ + # 还有就是由于本项目加载模型的方式设置的比较严格，下载完成后仍需手动修改模型的文件名 + # 将其设置为与Huggface Hub一致的文件名 + # 此外不同时期的ggml格式并不兼容，因此不同时期的ggml需要安装不同的llama-cpp-python库，且实测pip install 不好使 + # 需要手动从https://github.com/abetlen/llama-cpp-python/releases/tag/下载对应的wheel安装 + # 实测v0.1.63与本模型的vicuna/ggml-vicuna-13b-1.1/ggml-vic13b-q5_1.bin可以兼容 + "local_model_path": f'''{"/".join(os.path.abspath(__file__).split("/")[:3])}/.cache/huggingface/hub/models--vicuna--ggml-vicuna-13b-1.1/blobs/''', + "provides": "LLamaLLMChain" }, # 通过 fastchat 调用的模型请参考如下格式 @@ -83,7 +151,24 @@ llm_model_dict = { "name": "chatglm-6b", # "name"修改为fastchat服务中的"model_name" "pretrained_model_name": "chatglm-6b", "local_model_path": None, - "provides": "FastChatOpenAILLM", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLM" + "provides": "FastChatOpenAILLMChain", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLMChain" + "api_base_url": "http://localhost:8000/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + # 通过 fastchat 调用的模型请参考如下格式 + "fastchat-chatglm-6b-int4": { + "name": "chatglm-6b-int4", # "name"修改为fastchat服务中的"model_name" + "pretrained_model_name": "chatglm-6b-int4", + "local_model_path": None, + "provides": "FastChatOpenAILLMChain", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLMChain" + "api_base_url": "http://localhost:8001/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + "fastchat-chatglm2-6b": { + "name": "chatglm2-6b", # "name"修改为fastchat服务中的"model_name" + "pretrained_model_name": "chatglm2-6b", + "local_model_path": None, + "provides": "FastChatOpenAILLMChain", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLMChain" "api_base_url": "http://localhost:8000/v1" # "name"修改为fastchat服务中的"api_base_url" }, @@ -92,9 +177,29 @@ llm_model_dict = { "name": "vicuna-13b-hf", # "name"修改为fastchat服务中的"model_name" "pretrained_model_name": "vicuna-13b-hf", "local_model_path": None, - "provides": "FastChatOpenAILLM", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLM" - "api_base_url": "http://localhost:8000/v1" # "name"修改为fastchat服务中的"api_base_url" + "provides": "FastChatOpenAILLMChain", # 使用fastchat api时，需保证"provides"为"FastChatOpenAILLMChain" + "api_base_url": "http://localhost:8000/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" }, + # 调用chatgpt时如果报出： urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.openai.com', port=443): + # Max retries exceeded with url: /v1/chat/completions + # 则需要将urllib3版本修改为1.25.11 + # 如果依然报urllib3.exceptions.MaxRetryError: HTTPSConnectionPool，则将https改为http + # 参考https://zhuanlan.zhihu.com/p/350015032 + + # 如果报出：raise NewConnectionError( + # urllib3.exceptions.NewConnectionError: : + # Failed to establish a new connection: [WinError 10060] + # 则是因为内地和香港的IP都被OPENAI封了，需要切换为日本、新加坡等地 + "openai-chatgpt-3.5": { + "name": "gpt-3.5-turbo", + "pretrained_model_name": "gpt-3.5-turbo", + "provides": "FastChatOpenAILLMChain", + "local_model_path": None, + "api_base_url": "https://api.openapi.com/v1", + "api_key": "" + }, + } # LLM 名称 @@ -115,7 +220,7 @@ STREAMING = True # Use p-tuning-v2 PrefixEncoder USE_PTUNING_V2 = False - +PTUNING_DIR='./ptuning-v2' # LLM running device LLM_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" @@ -128,7 +233,7 @@ PROMPT_TEMPLATE = """已知信息： 根据上述已知信息，简洁和专业的来回答用户的问题。如果无法从中得到答案，请说 “根据已知信息无法回答该问题” 或 “没有提供足够的相关信息”，不允许在答案中添加编造成分，答案请使用中文。 问题是：{question}""" -# 缓存知识库数量 +# 缓存知识库数量,如果是ChatGLM2,ChatGLM2-int4,ChatGLM2-int8模型若检索效果不好可以调成’10’ CACHED_VS_NUM = 1 # 文本分句长度 @@ -143,8 +248,8 @@ LLM_HISTORY_LEN = 3 # 知识库检索时返回的匹配内容条数 VECTOR_SEARCH_TOP_K = 5 -# 知识检索内容相关度 Score, 数值范围约为0-1100，如果为0，则不生效，经测试设置为小于500时，匹配结果更精准 -VECTOR_SEARCH_SCORE_THRESHOLD = 0 +# 知识检索内容相关度 Score, 数值范围约为0-1100，如果为0，则不生效，建议设置为500左右，经测试设置为小于500时，匹配结果更精准 +VECTOR_SEARCH_SCORE_THRESHOLD = 500 NLTK_DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "nltk_data") From 310a865ccd359b8cc93ecb02cb2e7266938a19f2 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Thu, 27 Jul 2023 14:26:50 +0800 Subject: [PATCH 12/14] =?UTF-8?q?=E5=88=A0=E9=99=A4requirements=E7=9A=84?= =?UTF-8?q?=E4=B8=AD=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7f97f67f..84adf2ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,12 +24,6 @@ openai #peft~=0.3.0 #bitsandbytes; platform_system != "Windows" -# 要调用llama-cpp模型，如vicuma-13b量化模型需要安装llama-cpp-python库 -# but!!! 实测pip install 不好使，需要手动从ttps://github.com/abetlen/llama-cpp-python/releases/下载 -# 而且注意不同时期的ggml格式并不！兼！容!!!因此需要安装的llama-cpp-python版本也不一致，需要手动测试才能确定 -# 实测ggml-vicuna-13b-1.1在llama-cpp-python 0.1.63上可正常兼容 -# 不过！！！本项目模型加载的方式控制的比较严格，与llama-cpp-python的兼容性较差，很多参数设定不能使用， -# 建议如非必要还是不要使用llama-cpp torch~=2.0.0 pydantic~=1.10.7 starlette~=0.26.1 From c431bee94184beb619530039e8dd508168cfaf25 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Thu, 27 Jul 2023 14:27:11 +0800 Subject: [PATCH 13/14] =?UTF-8?q?=E5=88=A0=E9=99=A4requirements=E7=9A=84?= =?UTF-8?q?=E4=B8=AD=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/loader/loader.py | 7 +++++++ requirements.txt | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/models/loader/loader.py b/models/loader/loader.py index f43bb1d2..c5c80fdd 100644 --- a/models/loader/loader.py +++ b/models/loader/loader.py @@ -177,6 +177,13 @@ class LoaderCheckPoint: ) elif self.is_llamacpp: + + # 要调用llama-cpp模型，如vicuma-13b量化模型需要安装llama-cpp-python库 + # but!!! 实测pip install 不好使，需要手动从ttps://github.com/abetlen/llama-cpp-python/releases/下载 + # 而且注意不同时期的ggml格式并不！兼！容!!!因此需要安装的llama-cpp-python版本也不一致，需要手动测试才能确定 + # 实测ggml-vicuna-13b-1.1在llama-cpp-python 0.1.63上可正常兼容 + # 不过！！！本项目模型加载的方式控制的比较严格，与llama-cpp-python的兼容性较差，很多参数设定不能使用， + # 建议如非必要还是不要使用llama-cpp try: from llama_cpp import Llama diff --git a/requirements.txt b/requirements.txt index 84adf2ac..4c981b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,6 @@ openai #accelerate~=0.18.0 #peft~=0.3.0 #bitsandbytes; platform_system != "Windows" - torch~=2.0.0 pydantic~=1.10.7 starlette~=0.26.1 From 161c9e6c509278768a2ddd9db53ba67287e951b7 Mon Sep 17 00:00:00 2001 From: hzg0601 Date: Fri, 28 Jul 2023 03:59:29 +0800 Subject: [PATCH 14/14] fix typo --- models/loader/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/loader/args.py b/models/loader/args.py index 02f49269..a62b4cff 100644 --- a/models/loader/args.py +++ b/models/loader/args.py @@ -45,7 +45,7 @@ parser.add_argument('--model-name', type=str, default=LLM_MODEL, help='Name of t parser.add_argument("--use-lora",type=bool,default=USE_LORA,help="use lora or not") parser.add_argument('--lora', type=str, default=LORA_NAME,help='Name of the LoRA to apply to the model by default.') parser.add_argument("--lora-dir", type=str, default=LORA_DIR, help="Path to directory with all the loras") -parser.add_argument('--use-ptuning-v2',action=USE_PTUNING_V2,help="whether use ptuning-v2 checkpoint") +parser.add_argument('--use-ptuning-v2',default=USE_PTUNING_V2,help="whether use ptuning-v2 checkpoint") parser.add_argument("--ptuning-dir",type=str,default=PTUNING_DIR,help="the dir of ptuning-v2 checkpoint") # Accelerate/transformers parser.add_argument('--load-in-8bit', action='store_true', default=LOAD_IN_8BIT,