RYDE-WORK/Langchain-Chatchat
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/Langchain-Chatchat.git synced 2026-01-19 13:23:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

支持通过配置项同时启动多个模型，将Wiki纳入samples知识库 (#2002)

Browse Source 
新功能：
- 将 LLM_MODEL 配置项改为 LLM_MODELS 列表，同时启动多个模型
- 将 wiki 纳入 samples 知识库

依赖变化：
- 指定 streamlit~=1.27.0。1.26.0会报rerun错误，1.28.0会有无限刷新错误

修复优化：
- 优化 get_default_llm_model 逻辑
- 适配 Qwen 在线 API 做 Embeddings 时最大 25 行的限制
- 列出知识库磁盘文件时跳过 . 开头的文件
This commit is contained in:
liunux4odoo 2023-11-09 22:15:52 +08:00 committed by GitHub
parent ce1001a043
commit b51ba11f45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 238 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -2,7 +2,10 @@
*.log.*
*.bak
logs
/knowledge_base/
/knowledge_base/*
!/knowledge_base/samples
/knowledge_base/samples/vector_store
/configs/*.py
.vscode/

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "knowledge_base/samples/content/wiki"]
path = knowledge_base/samples/content/wiki
url = https://github.com/chatchat-space/Langchain-Chatchat.wiki.git

4
chains/llmchain_with_history.py
View File

@ -1,12 +1,12 @@
from server.utils import get_ChatOpenAI
from configs.model_config import LLM_MODEL, TEMPERATURE
from configs.model_config import LLM_MODELS, TEMPERATURE
from langchain.chains import LLMChain
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
)
model = get_ChatOpenAI(model_name=LLM_MODEL, temperature=TEMPERATURE)
model = get_ChatOpenAI(model_name=LLM_MODELS[0], temperature=TEMPERATURE)
human_prompt = "{input}"

1
configs/basic_config.py.example
View File

@ -2,6 +2,7 @@ import logging
import os
import langchain
# 是否显示详细日志
log_verbose = False
langchain.verbose = False

3
configs/kb_config.py.example
View File

@ -56,7 +56,10 @@ KB_INFO = {
"知识库名称": "知识库介绍",
"samples": "关于本项目issue的解答",
}
# 通常情况下不需要更改以下内容
# 知识库默认存储路径
KB_ROOT_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "knowledge_base")
if not os.path.exists(KB_ROOT_PATH):

187
configs/model_config.py.example
View File

@ -1,96 +1,13 @@
import os
# 可以指定一个绝对路径，统一存放所有的Embedding和LLM模型。
# 每个模型可以是一个单独的目录，也可以是某个目录下的二级子目录
# 每个模型可以是一个单独的目录，也可以是某个目录下的二级子目录。
# 如果模型目录名称和 MODEL_PATH 中的 key 或 value 相同，程序会自动检测加载，无需修改 MODEL_PATH 中的路径。
MODEL_ROOT_PATH = ""
# 在以下字典中修改属性值，以指定本地embedding模型存储位置。支持3种设置方法：
# 1、将对应的值修改为模型绝对路径
# 2、不修改此处的值（以 text2vec 为例）：
# 2.1 如果{MODEL_ROOT_PATH}下存在如下任一子目录：
# - text2vec
# - GanymedeNil/text2vec-large-chinese
# - text2vec-large-chinese
# 2.2 如果以上本地路径不存在，则使用huggingface模型
MODEL_PATH = {
"embed_model": {
"ernie-tiny": "nghuyong/ernie-3.0-nano-zh",
"ernie-base": "nghuyong/ernie-3.0-base-zh",
"text2vec-base": "shibing624/text2vec-base-chinese",
"text2vec": "GanymedeNil/text2vec-large-chinese",
"text2vec-paraphrase": "shibing624/text2vec-base-chinese-paraphrase",
"text2vec-sentence": "shibing624/text2vec-base-chinese-sentence",
"text2vec-multilingual": "shibing624/text2vec-base-multilingual",
"text2vec-bge-large-chinese": "shibing624/text2vec-bge-large-chinese",
"m3e-small": "moka-ai/m3e-small",
"m3e-base": "moka-ai/m3e-base",
"m3e-large": "moka-ai/m3e-large",
"bge-small-zh": "BAAI/bge-small-zh",
"bge-base-zh": "BAAI/bge-base-zh",
"bge-large-zh": "BAAI/bge-large-zh",
"bge-large-zh-noinstruct": "BAAI/bge-large-zh-noinstruct",
"bge-base-zh-v1.5": "BAAI/bge-base-zh-v1.5",
"bge-large-zh-v1.5": "BAAI/bge-large-zh-v1.5",
"piccolo-base-zh": "sensenova/piccolo-base-zh",
"piccolo-large-zh": "sensenova/piccolo-large-zh",
"text-embedding-ada-002": "your OPENAI_API_KEY",
},
# TODO: add all supported llm models
"llm_model": {
# 以下部分模型并未完全测试，仅根据fastchat和vllm模型的模型列表推定支持
"chatglm2-6b": "THUDM/chatglm2-6b",
"chatglm2-6b-32k": "THUDM/chatglm2-6b-32k",
"chatglm3-6b": "THUDM/chatglm3-6b-32k",
"chatglm3-6b-32k": "THUDM/chatglm3-6b-32k",
"baichuan2-13b": "baichuan-inc/Baichuan2-13B-Chat",
"baichuan2-7b": "baichuan-inc/Baichuan2-7B-Chat",
"baichuan-7b": "baichuan-inc/Baichuan-7B",
"baichuan-13b": "baichuan-inc/Baichuan-13B",
'baichuan-13b-chat': 'baichuan-inc/Baichuan-13B-Chat',
"aquila-7b": "BAAI/Aquila-7B",
"aquilachat-7b": "BAAI/AquilaChat-7B",
"internlm-7b": "internlm/internlm-7b",
"internlm-chat-7b": "internlm/internlm-chat-7b",
"falcon-7b": "tiiuae/falcon-7b",
"falcon-40b": "tiiuae/falcon-40b",
"falcon-rw-7b": "tiiuae/falcon-rw-7b",
"gpt2": "gpt2",
"gpt2-xl": "gpt2-xl",
"gpt-j-6b": "EleutherAI/gpt-j-6b",
"gpt4all-j": "nomic-ai/gpt4all-j",
"gpt-neox-20b": "EleutherAI/gpt-neox-20b",
"pythia-12b": "EleutherAI/pythia-12b",
"oasst-sft-4-pythia-12b-epoch-3.5": "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
"dolly-v2-12b": "databricks/dolly-v2-12b",
"stablelm-tuned-alpha-7b": "stabilityai/stablelm-tuned-alpha-7b",
"Llama-2-13b-hf": "meta-llama/Llama-2-13b-hf",
"Llama-2-70b-hf": "meta-llama/Llama-2-70b-hf",
"open_llama_13b": "openlm-research/open_llama_13b",
"vicuna-13b-v1.3": "lmsys/vicuna-13b-v1.3",
"koala": "young-geng/koala",
"mpt-7b": "mosaicml/mpt-7b",
"mpt-7b-storywriter": "mosaicml/mpt-7b-storywriter",
"mpt-30b": "mosaicml/mpt-30b",
"opt-66b": "facebook/opt-66b",
"opt-iml-max-30b": "facebook/opt-iml-max-30b",
"Qwen-7B": "Qwen/Qwen-7B",
"Qwen-14B": "Qwen/Qwen-14B",
"Qwen-7B-Chat": "Qwen/Qwen-7B-Chat",
"Qwen-14B-Chat": "Qwen/Qwen-14B-Chat",
},
}
# 选用的 Embedding 名称
EMBEDDING_MODEL = "m3e-base" # 可以尝试最新的嵌入式sota模型：bge-large-zh-v1.5
EMBEDDING_MODEL = "m3e-base" # bge-large-zh
# Embedding 模型运行设备。设为"auto"会自动检测，也可手动设定为"cuda","mps","cpu"其中之一。
EMBEDDING_DEVICE = "auto"
@ -99,9 +16,11 @@ EMBEDDING_DEVICE = "auto"
EMBEDDING_KEYWORD_FILE = "keywords.txt"
EMBEDDING_MODEL_OUTPUT_PATH = "output"
# LLM 名称
LLM_MODEL = "chatglm2-6b"
# AgentLM模型的名称 (可以不指定，指定之后就锁定进入Agent之后的Chain的模型，不指定就是LLM_MODEL)
# 要运行的 LLM 名称，可以包括本地模型和在线模型。
# 第一个将作为 API 和 WEBUI 的默认模型
LLM_MODELS = ["chatglm2-6b-int4", "zhipu-api", "openai-api]
# AgentLM模型的名称 (可以不指定，指定之后就锁定进入Agent之后的Chain的模型，不指定就是LLM_MODELS[0])
Agent_MODEL = None
# LLM 运行设备。设为"auto"会自动检测，也可手动设定为"cuda","mps","cpu"其中之一。
@ -111,7 +30,6 @@ LLM_DEVICE = "auto"
HISTORY_LEN = 3
# 大模型最长支持的长度，如果不填写，则使用模型默认的最大长度，如果填写，则为用户设定的最大长度
MAX_TOKENS = None
# LLM通用对话参数
@ -197,6 +115,93 @@ ONLINE_LLM_MODEL = {
},
}
# 在以下字典中修改属性值，以指定本地embedding模型存储位置。支持3种设置方法：
# 1、将对应的值修改为模型绝对路径
# 2、不修改此处的值（以 text2vec 为例）：
# 2.1 如果{MODEL_ROOT_PATH}下存在如下任一子目录：
# - text2vec
# - GanymedeNil/text2vec-large-chinese
# - text2vec-large-chinese
# 2.2 如果以上本地路径不存在，则使用huggingface模型
MODEL_PATH = {
"embed_model": {
"ernie-tiny": "nghuyong/ernie-3.0-nano-zh",
"ernie-base": "nghuyong/ernie-3.0-base-zh",
"text2vec-base": "shibing624/text2vec-base-chinese",
"text2vec": "GanymedeNil/text2vec-large-chinese",
"text2vec-paraphrase": "shibing624/text2vec-base-chinese-paraphrase",
"text2vec-sentence": "shibing624/text2vec-base-chinese-sentence",
"text2vec-multilingual": "shibing624/text2vec-base-multilingual",
"text2vec-bge-large-chinese": "shibing624/text2vec-bge-large-chinese",
"m3e-small": "moka-ai/m3e-small",
"m3e-base": "moka-ai/m3e-base",
"m3e-large": "moka-ai/m3e-large",
"bge-small-zh": "BAAI/bge-small-zh",
"bge-base-zh": "BAAI/bge-base-zh",
"bge-large-zh": "BAAI/bge-large-zh",
"bge-large-zh-noinstruct": "BAAI/bge-large-zh-noinstruct",
"bge-base-zh-v1.5": "BAAI/bge-base-zh-v1.5",
"bge-large-zh-v1.5": "BAAI/bge-large-zh-v1.5",
"piccolo-base-zh": "sensenova/piccolo-base-zh",
"piccolo-large-zh": "sensenova/piccolo-large-zh",
"text-embedding-ada-002": "your OPENAI_API_KEY",
},
"llm_model": {
# 以下部分模型并未完全测试，仅根据fastchat和vllm模型的模型列表推定支持
"chatglm2-6b": "THUDM/chatglm2-6b",
"chatglm2-6b-32k": "THUDM/chatglm2-6b-32k",
"chatglm3-6b": "THUDM/chatglm3-6b-32k",
"chatglm3-6b-32k": "THUDM/chatglm3-6b-32k",
"baichuan2-13b": "baichuan-inc/Baichuan2-13B-Chat",
"baichuan2-7b": "baichuan-inc/Baichuan2-7B-Chat",
"baichuan-7b": "baichuan-inc/Baichuan-7B",
"baichuan-13b": "baichuan-inc/Baichuan-13B",
'baichuan-13b-chat': 'baichuan-inc/Baichuan-13B-Chat',
"aquila-7b": "BAAI/Aquila-7B",
"aquilachat-7b": "BAAI/AquilaChat-7B",
"internlm-7b": "internlm/internlm-7b",
"internlm-chat-7b": "internlm/internlm-chat-7b",
"falcon-7b": "tiiuae/falcon-7b",
"falcon-40b": "tiiuae/falcon-40b",
"falcon-rw-7b": "tiiuae/falcon-rw-7b",
"gpt2": "gpt2",
"gpt2-xl": "gpt2-xl",
"gpt-j-6b": "EleutherAI/gpt-j-6b",
"gpt4all-j": "nomic-ai/gpt4all-j",
"gpt-neox-20b": "EleutherAI/gpt-neox-20b",
"pythia-12b": "EleutherAI/pythia-12b",
"oasst-sft-4-pythia-12b-epoch-3.5": "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
"dolly-v2-12b": "databricks/dolly-v2-12b",
"stablelm-tuned-alpha-7b": "stabilityai/stablelm-tuned-alpha-7b",
"Llama-2-13b-hf": "meta-llama/Llama-2-13b-hf",
"Llama-2-70b-hf": "meta-llama/Llama-2-70b-hf",
"open_llama_13b": "openlm-research/open_llama_13b",
"vicuna-13b-v1.3": "lmsys/vicuna-13b-v1.3",
"koala": "young-geng/koala",
"mpt-7b": "mosaicml/mpt-7b",
"mpt-7b-storywriter": "mosaicml/mpt-7b-storywriter",
"mpt-30b": "mosaicml/mpt-30b",
"opt-66b": "facebook/opt-66b",
"opt-iml-max-30b": "facebook/opt-iml-max-30b",
"Qwen-7B": "Qwen/Qwen-7B",
"Qwen-14B": "Qwen/Qwen-14B",
"Qwen-7B-Chat": "Qwen/Qwen-7B-Chat",
"Qwen-14B-Chat": "Qwen/Qwen-14B-Chat",
},
}
# 通常情况下不需要更改以下内容
# nltk 模型存储路径

50
configs/server_config.py.example
View File

@ -31,8 +31,7 @@ FSCHAT_OPENAI_API = {
# fastchat model_worker server
# 这些模型必须是在model_config.MODEL_PATH或ONLINE_MODEL中正确配置的。
# 在启动startup.py时，可用通过`--model-worker --model-name xxxx`指定模型，不指定则为LLM_MODEL
# 必须在这里添加的模型才会出现在WEBUI中可选模型列表里（LLM_MODEL会自动添加）
# 在启动startup.py时，可用通过`--model-name xxxx yyyy`指定模型，不指定则为LLM_MODELS
FSCHAT_MODEL_WORKERS = {
# 所有模型共用的默认配置，可在模型专项配置中进行覆盖。
"default": {
@ -58,7 +57,7 @@ FSCHAT_MODEL_WORKERS = {
# "awq_ckpt": None,
# "awq_wbits": 16,
# "awq_groupsize": -1,
# "model_names": [LLM_MODEL],
# "model_names": LLM_MODELS,
# "conv_template": None,
# "limit_worker_concurrency": 5,
# "stream_interval": 2,
@ -96,30 +95,31 @@ FSCHAT_MODEL_WORKERS = {
# "device": "cpu",
# },
"zhipu-api": { # 请为每个要运行的在线API设置不同的端口
#以下配置可以不用修改，在model_config中设置启动的模型
"zhipu-api": {
"port": 21001,
},
# "minimax-api": {
# "port": 21002,
# },
# "xinghuo-api": {
# "port": 21003,
# },
# "qianfan-api": {
# "port": 21004,
# },
# "fangzhou-api": {
# "port": 21005,
# },
# "qwen-api": {
# "port": 21006,
# },
# "baichuan-api": {
# "port": 21007,
# },
# "azure-api": {
# "port": 21008,
# },
"minimax-api": {
"port": 21002,
},
"xinghuo-api": {
"port": 21003,
},
"qianfan-api": {
"port": 21004,
},
"fangzhou-api": {
"port": 21005,
},
"qwen-api": {
"port": 21006,
},
"baichuan-api": {
"port": 21007,
},
"azure-api": {
"port": 21008,
},
}
# fastchat multi model worker server

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_closed.csv → knowledge_base/samples/content/test_files/langchain-ChatGLM_closed.csv
View File

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_closed.jsonl → knowledge_base/samples/content/test_files/langchain-ChatGLM_closed.jsonl
View File

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_closed.xlsx → knowledge_base/samples/content/test_files/langchain-ChatGLM_closed.xlsx
View File

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_open.csv → knowledge_base/samples/content/test_files/langchain-ChatGLM_open.csv
View File

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_open.jsonl → knowledge_base/samples/content/test_files/langchain-ChatGLM_open.jsonl
View File

0
knowledge_base/samples/isssues_merge/langchain-ChatGLM_open.xlsx → knowledge_base/samples/content/test_files/langchain-ChatGLM_open.xlsx
View File

0
knowledge_base/samples/content/test.txt → knowledge_base/samples/content/test_files/test.txt
View File

1
knowledge_base/samples/content/wiki Submodule

@ -0,0 +1 @@
Subproject commit b705cf80e4150cb900c77b343f0f9c62ec9a0278

2
requirements.txt
View File

@ -53,7 +53,7 @@ vllm>=0.2.0; sys_platform == "linux"
# WebUI requirements
streamlit>=1.26.0
streamlit~=1.27.0
streamlit-option-menu>=0.3.6
streamlit-antd-components>=0.1.11
streamlit-chatbox>=1.1.11

2
requirements_lite.txt
View File

@ -41,7 +41,7 @@ dashscope>=1.10.0 # qwen
numpy~=1.24.4
pandas~=2.0.3
streamlit>=1.26.0
streamlit~=1.27.0
streamlit-option-menu>=0.3.6
streamlit-antd-components>=0.1.11
streamlit-chatbox==1.1.11

2
requirements_webui.txt
View File

@ -1,6 +1,6 @@
# WebUI requirements
streamlit>=1.26.0
streamlit~=1.27.0
streamlit-option-menu>=0.3.6
streamlit-antd-components>=0.1.11
streamlit-chatbox>=1.1.11

6
server/chat/agent_chat.py
View File

@ -5,7 +5,7 @@ from langchain.agents import AgentExecutor, LLMSingleActionAgent, initialize_age
from server.agent.custom_template import CustomOutputParser, CustomPromptTemplate
from fastapi import Body
from fastapi.responses import StreamingResponse
from configs import LLM_MODEL, TEMPERATURE, HISTORY_LEN, Agent_MODEL
from configs import LLM_MODELS, TEMPERATURE, HISTORY_LEN, Agent_MODEL
from server.utils import wrap_done, get_ChatOpenAI, get_prompt_template
from langchain.chains import LLMChain
from typing import AsyncIterable, Optional, Dict
@ -26,7 +26,7 @@ async def agent_chat(query: str = Body(..., description="用户输入", examples
"content": "使用天气查询工具查询到今天北京多云，10-14摄氏度，东北风2级，易感冒"}]]
),
stream: bool = Body(False, description="流式输出"),
model_name: str = Body(LLM_MODEL, description="LLM 模型名称。"),
model_name: str = Body(LLM_MODELS[0], description="LLM 模型名称。"),
temperature: float = Body(TEMPERATURE, description="LLM 采样温度", ge=0.0, le=1.0),
max_tokens: Optional[int] = Body(None, description="限制LLM生成Token数量，默认None代表模型最大值"),
prompt_name: str = Body("default",
@ -38,7 +38,7 @@ async def agent_chat(query: str = Body(..., description="用户输入", examples
async def agent_chat_iterator(
query: str,
history: Optional[List[History]],
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
prompt_name: str = prompt_name,
) -> AsyncIterable[str]:
callback = CustomAsyncIteratorCallbackHandler()

6
server/chat/chat.py
View File

@ -1,6 +1,6 @@
from fastapi import Body
from fastapi.responses import StreamingResponse
from configs import LLM_MODEL, TEMPERATURE, SAVE_CHAT_HISTORY
from configs import LLM_MODELS, TEMPERATURE, SAVE_CHAT_HISTORY
from server.utils import wrap_done, get_ChatOpenAI
from langchain.chains import LLMChain
from langchain.callbacks import AsyncIteratorCallbackHandler
@ -22,7 +22,7 @@ async def chat(query: str = Body(..., description="用户输入", examples=["恼
{"role": "assistant", "content": "虎头虎脑"}]]
),
stream: bool = Body(False, description="流式输出"),
model_name: str = Body(LLM_MODEL, description="LLM 模型名称。"),
model_name: str = Body(LLM_MODELS[0], description="LLM 模型名称。"),
temperature: float = Body(TEMPERATURE, description="LLM 采样温度", ge=0.0, le=1.0),
max_tokens: Optional[int] = Body(None, description="限制LLM生成Token数量，默认None代表模型最大值"),
# top_p: float = Body(TOP_P, description="LLM 核采样。勿与temperature同时设置", gt=0.0, lt=1.0),
@ -32,7 +32,7 @@ async def chat(query: str = Body(..., description="用户输入", examples=["恼
async def chat_iterator(query: str,
history: List[History] = [],
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
prompt_name: str = prompt_name,
) -> AsyncIterable[str]:
callback = AsyncIteratorCallbackHandler()

6
server/chat/completion.py
View File

@ -1,6 +1,6 @@
from fastapi import Body
from fastapi.responses import StreamingResponse
from configs import LLM_MODEL, TEMPERATURE
from configs import LLM_MODELS, TEMPERATURE
from server.utils import wrap_done, get_OpenAI
from langchain.chains import LLMChain
from langchain.callbacks import AsyncIteratorCallbackHandler
@ -13,7 +13,7 @@ from server.utils import get_prompt_template
async def completion(query: str = Body(..., description="用户输入", examples=["恼羞成怒"]),
stream: bool = Body(False, description="流式输出"),
echo: bool = Body(False, description="除了输出之外，还回显输入"),
model_name: str = Body(LLM_MODEL, description="LLM 模型名称。"),
model_name: str = Body(LLM_MODELS[0], description="LLM 模型名称。"),
temperature: float = Body(TEMPERATURE, description="LLM 采样温度", ge=0.0, le=1.0),
max_tokens: Optional[int] = Body(1024, description="限制LLM生成Token数量，默认None代表模型最大值"),
# top_p: float = Body(TOP_P, description="LLM 核采样。勿与temperature同时设置", gt=0.0, lt=1.0),
@ -23,7 +23,7 @@ async def completion(query: str = Body(..., description="用户输入", examples
#todo 因ApiModelWorker 默认是按chat处理的，会对params["prompt"] 解析为messages，因此ApiModelWorker 使用时需要有相应处理
async def completion_iterator(query: str,
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
prompt_name: str = prompt_name,
echo: bool = echo,
) -> AsyncIterable[str]:

6
server/chat/knowledge_base_chat.py
View File

@ -1,6 +1,6 @@
from fastapi import Body, Request
from fastapi.responses import StreamingResponse
from configs import (LLM_MODEL, VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD, TEMPERATURE)
from configs import (LLM_MODELS, VECTOR_SEARCH_TOP_K, SCORE_THRESHOLD, TEMPERATURE)
from server.utils import wrap_done, get_ChatOpenAI
from server.utils import BaseResponse, get_prompt_template
from langchain.chains import LLMChain
@ -30,7 +30,7 @@ async def knowledge_base_chat(query: str = Body(..., description="用户输入",
"content": "虎头虎脑"}]]
),
stream: bool = Body(False, description="流式输出"),
model_name: str = Body(LLM_MODEL, description="LLM 模型名称。"),
model_name: str = Body(LLM_MODELS[0], description="LLM 模型名称。"),
temperature: float = Body(TEMPERATURE, description="LLM 采样温度", ge=0.0, le=1.0),
max_tokens: Optional[int] = Body(None, description="限制LLM生成Token数量，默认None代表模型最大值"),
prompt_name: str = Body("default", description="使用的prompt模板名称(在configs/prompt_config.py中配置)"),
@ -45,7 +45,7 @@ async def knowledge_base_chat(query: str = Body(..., description="用户输入",
async def knowledge_base_chat_iterator(query: str,
top_k: int,
history: Optional[List[History]],
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
prompt_name: str = prompt_name,
) -> AsyncIterable[str]:
callback = AsyncIteratorCallbackHandler()

4
server/chat/openai_chat.py
View File

@ -1,7 +1,7 @@
from fastapi.responses import StreamingResponse
from typing import List, Optional
import openai
from configs import LLM_MODEL, logger, log_verbose
from configs import LLM_MODELS, logger, log_verbose
from server.utils import get_model_worker_config, fschat_openai_api_address
from pydantic import BaseModel
@ -12,7 +12,7 @@ class OpenAiMessage(BaseModel):
class OpenAiChatMsgIn(BaseModel):
model: str = LLM_MODEL
model: str = LLM_MODELS[0]
messages: List[OpenAiMessage]
temperature: float = 0.7
n: int = 1

6
server/chat/search_engine_chat.py
View File

@ -1,7 +1,7 @@
from langchain.utilities.bing_search import BingSearchAPIWrapper
from langchain.utilities.duckduckgo_search import DuckDuckGoSearchAPIWrapper
from configs import (BING_SEARCH_URL, BING_SUBSCRIPTION_KEY, METAPHOR_API_KEY,
LLM_MODEL, SEARCH_ENGINE_TOP_K, TEMPERATURE,
LLM_MODELS, SEARCH_ENGINE_TOP_K, TEMPERATURE,
TEXT_SPLITTER_NAME, OVERLAP_SIZE)
from fastapi import Body
from fastapi.responses import StreamingResponse
@ -126,7 +126,7 @@ async def search_engine_chat(query: str = Body(..., description="用户输入",
"content": "虎头虎脑"}]]
),
stream: bool = Body(False, description="流式输出"),
model_name: str = Body(LLM_MODEL, description="LLM 模型名称。"),
model_name: str = Body(LLM_MODELS[0], description="LLM 模型名称。"),
temperature: float = Body(TEMPERATURE, description="LLM 采样温度", ge=0.0, le=1.0),
max_tokens: Optional[int] = Body(None, description="限制LLM生成Token数量，默认None代表模型最大值"),
prompt_name: str = Body("default",description="使用的prompt模板名称(在configs/prompt_config.py中配置)"),
@ -144,7 +144,7 @@ async def search_engine_chat(query: str = Body(..., description="用户输入",
search_engine_name: str,
top_k: int,
history: Optional[List[History]],
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
prompt_name: str = prompt_name,
) -> AsyncIterable[str]:
callback = AsyncIteratorCallbackHandler()

12
server/knowledge_base/kb_service/faiss_kb_service.py
View File

@ -48,7 +48,10 @@ class FaissKBService(KBService):
def do_drop_kb(self):
self.clear_vs()
shutil.rmtree(self.kb_path)
try:
shutil.rmtree(self.kb_path)
except Exception:
...
def do_search(self,
query: str,
@ -90,8 +93,11 @@ class FaissKBService(KBService):
def do_clear_vs(self):
with kb_faiss_pool.atomic:
kb_faiss_pool.pop((self.kb_name, self.vector_name))
shutil.rmtree(self.vs_path)
os.makedirs(self.vs_path)
try:
shutil.rmtree(self.vs_path)
except Exception:
...
os.makedirs(self.vs_path, exist_ok=True)
def exist_doc(self, file_name: str):
if super().exist_doc(file_name):

7
server/knowledge_base/utils.py
View File

@ -7,7 +7,7 @@ from configs import (
logger,
log_verbose,
text_splitter_dict,
LLM_MODEL,
LLM_MODELS,
TEXT_SPLITTER_NAME,
)
import importlib
@ -57,7 +57,8 @@ def list_files_from_folder(kb_name: str):
for root, _, files in os.walk(doc_path):
tail = os.path.basename(root).lower()
if (tail.startswith("temp")
or tail.startswith("tmp")): # 跳过 temp 或 tmp 开头的文件夹
or tail.startswith("tmp")
or tail.startswith(".")): # 跳过 [temp, tmp, .] 开头的文件夹
continue
for file in files:
if file.startswith("~$"): # 跳过 ~$ 开头的文件
@ -192,7 +193,7 @@ def make_text_splitter(
splitter_name: str = TEXT_SPLITTER_NAME,
chunk_size: int = CHUNK_SIZE,
chunk_overlap: int = OVERLAP_SIZE,
llm_model: str = LLM_MODEL,
llm_model: str = LLM_MODELS[0],
):
"""
根据参数获取特定的分词器

8
server/llm_api.py
View File

@ -1,5 +1,5 @@
from fastapi import Body
from configs import logger, log_verbose, LLM_MODEL, HTTPX_DEFAULT_TIMEOUT
from configs import logger, log_verbose, LLM_MODELS, HTTPX_DEFAULT_TIMEOUT
from server.utils import (BaseResponse, fschat_controller_address, list_config_llm_models,
get_httpx_client, get_model_worker_config)
from copy import deepcopy
@ -65,7 +65,7 @@ def get_model_config(
def stop_llm_model(
model_name: str = Body(..., description="要停止的LLM模型名称", examples=[LLM_MODEL]),
model_name: str = Body(..., description="要停止的LLM模型名称", examples=[LLM_MODELS[0]]),
controller_address: str = Body(None, description="Fastchat controller服务器地址", examples=[fschat_controller_address()])
) -> BaseResponse:
'''
@ -89,8 +89,8 @@ def stop_llm_model(
def change_llm_model(
model_name: str = Body(..., description="当前运行模型", examples=[LLM_MODEL]),
new_model_name: str = Body(..., description="要切换的新模型", examples=[LLM_MODEL]),
model_name: str = Body(..., description="当前运行模型", examples=[LLM_MODELS[0]]),
new_model_name: str = Body(..., description="要切换的新模型", examples=[LLM_MODELS[0]]),
controller_address: str = Body(None, description="Fastchat controller服务器地址", examples=[fschat_controller_address()])
):
'''

26
server/model_workers/qwen.py
View File

@ -59,16 +59,22 @@ class QwenWorker(ApiModelWorker):
import dashscope
params.load_config(self.model_names[0])
resp = dashscope.TextEmbedding.call(
model=params.embed_model or self.DEFAULT_EMBED_MODEL,
input=params.texts, # 最大25行
api_key=params.api_key,
)
if resp["status_code"] != 200:
return {"code": resp["status_code"], "msg": resp.message}
else:
embeddings = [x["embedding"] for x in resp["output"]["embeddings"]]
return {"code": 200, "data": embeddings}
result = []
i = 0
while i < len(params.texts):
texts = params.texts[i:i+25]
resp = dashscope.TextEmbedding.call(
model=params.embed_model or self.DEFAULT_EMBED_MODEL,
input=texts, # 最大25行
api_key=params.api_key,
)
if resp["status_code"] != 200:
return {"code": resp["status_code"], "msg": resp.message}
else:
embeddings = [x["embedding"] for x in resp["output"]["embeddings"]]
result += embeddings
i += 25
return {"code": 200, "data": result}
def get_embeddings(self, params):
# TODO: 支持embeddings

9
server/utils.py
View File

@ -4,7 +4,7 @@ from typing import List
from fastapi import FastAPI
from pathlib import Path
import asyncio
from configs import (LLM_MODEL, LLM_DEVICE, EMBEDDING_DEVICE,
from configs import (LLM_MODELS, LLM_DEVICE, EMBEDDING_DEVICE,
MODEL_PATH, MODEL_ROOT_PATH, ONLINE_LLM_MODEL, logger, log_verbose,
FSCHAT_MODEL_WORKERS, HTTPX_DEFAULT_TIMEOUT)
import os
@ -345,8 +345,7 @@ def list_config_llm_models() -> Dict[str, Dict]:
return [(model_name, config_type), ...]
'''
workers = list(FSCHAT_MODEL_WORKERS)
if LLM_MODEL not in workers:
workers.insert(0, LLM_MODEL)
return {
"local": MODEL_PATH["llm_model"],
"online": ONLINE_LLM_MODEL,
@ -431,7 +430,7 @@ def fschat_controller_address() -> str:
return f"http://{host}:{port}"
def fschat_model_worker_address(model_name: str = LLM_MODEL) -> str:
def fschat_model_worker_address(model_name: str = LLM_MODELS[0]) -> str:
if model := get_model_worker_config(model_name): # TODO: depends fastchat
host = model["host"]
if host == "0.0.0.0":
@ -660,7 +659,7 @@ def get_server_configs() -> Dict:
TEXT_SPLITTER_NAME,
)
from configs.model_config import (
LLM_MODEL,
LLM_MODELS,
HISTORY_LEN,
TEMPERATURE,
)

14
startup.py
View File

@ -22,7 +22,7 @@ from configs import (
LOG_PATH,
log_verbose,
logger,
LLM_MODEL,
LLM_MODELS,
EMBEDDING_MODEL,
TEXT_SPLITTER_NAME,
FSCHAT_CONTROLLER,
@ -359,7 +359,7 @@ def run_controller(log_level: str = "INFO", started_event: mp.Event = None):
def run_model_worker(
model_name: str = LLM_MODEL,
model_name: str = LLM_MODELS[0],
controller_address: str = "",
log_level: str = "INFO",
q: mp.Queue = None,
@ -496,7 +496,7 @@ def parse_args() -> argparse.ArgumentParser:
"--model-worker",
action="store_true",
help="run fastchat's model_worker server with specified model name. "
"specify --model-name if not using default LLM_MODEL",
"specify --model-name if not using default LLM_MODELS",
dest="model_worker",
)
parser.add_argument(
@ -504,7 +504,7 @@ def parse_args() -> argparse.ArgumentParser:
"--model-name",
type=str,
nargs="+",
default=[LLM_MODEL],
default=LLM_MODELS,
help="specify model name for model worker. "
"add addition names with space seperated to start multiple model workers.",
dest="model_name",
@ -568,7 +568,7 @@ def dump_server_info(after_start=False, args=None):
print(f"langchain版本：{langchain.__version__}. fastchat版本：{fastchat.__version__}")
print("\n")
models = [LLM_MODEL]
models = LLM_MODELS
if args and args.model_name:
models = args.model_name
@ -694,8 +694,8 @@ async def start_main_server():
processes["model_worker"][model_name] = process
if args.api_worker:
configs = get_all_model_worker_configs()
for model_name, config in configs.items():
for model_name in args.model_name:
config = get_model_worker_config(model_name)
if (config.get("online_api")
and config.get("worker_class")
and model_name in FSCHAT_MODEL_WORKERS):

4
tests/agent/test_agent_function.py
View File

@ -1,7 +1,7 @@
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from configs import LLM_MODEL, TEMPERATURE
from configs import LLM_MODELS, TEMPERATURE
from server.utils import get_ChatOpenAI
from langchain.chains import LLMChain
from langchain.agents import LLMSingleActionAgent, AgentExecutor
@ -10,7 +10,7 @@ from langchain.memory import ConversationBufferWindowMemory
memory = ConversationBufferWindowMemory(k=5)
model = get_ChatOpenAI(
model_name=LLM_MODEL,
model_name=LLM_MODELS[0],
temperature=TEMPERATURE,
)
from server.agent.custom_template import CustomOutputParser, prompt

1
tests/api/test_llm_api.py
View File

@ -6,7 +6,6 @@ from pathlib import Path
root_path = Path(__file__).parent.parent.parent
sys.path.append(str(root_path))
from configs.server_config import FSCHAT_MODEL_WORKERS
from configs.model_config import LLM_MODEL
from server.utils import api_address, get_model_worker_config
from pprint import pprint

57
webui_pages/utils.py
View File

@ -8,7 +8,7 @@ from pathlib import Path
from configs import (
EMBEDDING_MODEL,
DEFAULT_VS_TYPE,
LLM_MODEL,
LLM_MODELS,
TEMPERATURE,
SCORE_THRESHOLD,
CHUNK_SIZE,
@ -259,7 +259,7 @@ class ApiRequest:
self,
messages: List[Dict],
stream: bool = True,
model: str = LLM_MODEL,
model: str = LLM_MODELS[0],
temperature: float = TEMPERATURE,
max_tokens: int = None,
**kwargs: Any,
@ -291,7 +291,7 @@ class ApiRequest:
query: str,
history: List[Dict] = [],
stream: bool = True,
model: str = LLM_MODEL,
model: str = LLM_MODELS[0],
temperature: float = TEMPERATURE,
max_tokens: int = None,
prompt_name: str = "default",
@ -321,7 +321,7 @@ class ApiRequest:
query: str,
history: List[Dict] = [],
stream: bool = True,
model: str = LLM_MODEL,
model: str = LLM_MODELS[0],
temperature: float = TEMPERATURE,
max_tokens: int = None,
prompt_name: str = "default",
@ -353,7 +353,7 @@ class ApiRequest:
score_threshold: float = SCORE_THRESHOLD,
history: List[Dict] = [],
stream: bool = True,
model: str = LLM_MODEL,
model: str = LLM_MODELS[0],
temperature: float = TEMPERATURE,
max_tokens: int = None,
prompt_name: str = "default",
@ -391,7 +391,7 @@ class ApiRequest:
top_k: int = SEARCH_ENGINE_TOP_K,
history: List[Dict] = [],
stream: bool = True,
model: str = LLM_MODEL,
model: str = LLM_MODELS[0],
temperature: float = TEMPERATURE,
max_tokens: int = None,
prompt_name: str = "default",
@ -677,9 +677,10 @@ class ApiRequest:
return self._get_response_value(response, as_json=True, value_func=lambda r:r.get("data", []))
def get_default_llm_model(self) -> Tuple[str, bool]:
def get_default_llm_model(self, local_first: bool = True) -> Tuple[str, bool]:
'''
从服务器上获取当前运行的LLM模型，如果本机配置的LLM_MODEL属于本地模型且在其中，则优先返回
从服务器上获取当前运行的LLM模型。
当 local_first=True 时，优先返回运行中的本地模型，否则优先按LLM_MODELS配置顺序返回。
返回类型为（model_name, is_local_model）
'''
def ret_sync():
@ -687,26 +688,42 @@ class ApiRequest:
if not running_models:
return "", False
if LLM_MODEL in running_models:
return LLM_MODEL, True
model = ""
for m in LLM_MODELS:
if m not in running_models:
continue
is_local = not running_models[m].get("online_api")
if local_first and not is_local:
continue
else:
model = m
break
local_models = [k for k, v in running_models.items() if not v.get("online_api")]
if local_models:
return local_models[0], True
return list(running_models)[0], False
if not model: # LLM_MODELS中配置的模型都不在running_models里
model = list(running_models)[0]
is_local = not running_models[model].get("online_api")
return model, is_local
async def ret_async():
running_models = await self.list_running_models()
if not running_models:
return "", False
if LLM_MODEL in running_models:
return LLM_MODEL, True
model = ""
for m in LLM_MODELS:
if m not in running_models:
continue
is_local = not running_models[m].get("online_api")
if local_first and not is_local:
continue
else:
model = m
break
local_models = [k for k, v in running_models.items() if not v.get("online_api")]
if local_models:
return local_models[0], True
return list(running_models)[0], False
if not model: # LLM_MODELS中配置的模型都不在running_models里
model = list(running_models)[0]
is_local = not running_models[model].get("online_api")
return model, is_local
if self._use_async:
return ret_async()