mirror of
https://github.com/RYDE-WORK/Langchain-Chatchat.git
synced 2026-01-19 21:37:20 +08:00
xinference环境配置手册
This commit is contained in:
glide-the
parent
f423d80dd6
commit
47ac8628bd
162
libs/chatchat-server/README_xinference.md
Normal file
162
libs/chatchat-server/README_xinference.md
Normal file
@ -0,0 +1,162 @@
|
||||
|
||||
#### xinference环境配置手册
|
||||
|
||||
- 初始化conda
|
||||
```shell
|
||||
$ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
|
||||
$ rm -rf ~/miniconda3/
|
||||
$ bash Miniconda3-latest-Linux-x86_64.sh
|
||||
$ conda config --remove channels https://mirrors.ustc.edu.cn/anaconda/pkgs/main/
|
||||
$ conda config --remove channels https://mirrors.ustc.edu.cn/anaconda/pkgs/free/
|
||||
$ conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free
|
||||
$ conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
|
||||
|
||||
```
|
||||
|
||||
- 创建chatchat环境
|
||||
```shell
|
||||
$ conda create -p ~/miniconda3/envs/chatchat python=3.8
|
||||
$ conda activate ~/miniconda3/envs/chatchat
|
||||
$ pip install langchain-chatchat -U
|
||||
$ pip install xinference_client faiss-gpu "unstructured[pdf]"
|
||||
```
|
||||
|
||||
- 创建xinference环境
|
||||
```shell
|
||||
$ conda create -p ~/miniconda3/envs/xinference python=3.8
|
||||
$ conda activate ~/miniconda3/envs/xinference
|
||||
$ pip install xinference --force
|
||||
```
|
||||
|
||||
- 启动xinference服务
|
||||
```shell
|
||||
|
||||
$ xinference-local
|
||||
```
|
||||
- 编辑注册模型脚本
|
||||
```shell
|
||||
$ vim model_registrations.sh
|
||||
# 添加以下内容。模型路径需要根据实际情况修改
|
||||
curl 'http://127.0.0.1:9997/v1/model_registrations/LLM' \
|
||||
-H 'Accept: */*' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Cookie: token=no_auth' \
|
||||
-H 'Origin: http://127.0.0.1:9997' \
|
||||
-H 'Referer: http://127.0.0.1:9997/ui/' \
|
||||
-H 'Sec-Fetch-Dest: empty' \
|
||||
-H 'Sec-Fetch-Mode: cors' \
|
||||
-H 'Sec-Fetch-Site: same-origin' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \
|
||||
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
|
||||
-H 'sec-ch-ua-mobile: ?0' \
|
||||
-H 'sec-ch-ua-platform: "Linux"' \
|
||||
--data-raw '{"model":"{\"version\":1,\"model_name\":\"autodl-tmp-glm-4-9b-chat\",\"model_description\":\"autodl-tmp-glm-4-9b-chat\",\"context_length\":2048,\"model_lang\":[\"en\",\"zh\"],\"model_ability\":[\"generate\",\"chat\"],\"model_family\":\"glm4-chat\",\"model_specs\":[{\"model_uri\":\"/root/autodl-tmp/glm-4-9b-chat\",\"model_size_in_billions\":9,\"model_format\":\"pytorch\",\"quantizations\":[\"none\"]}],\"prompt_style\":{\"style_name\":\"CHATGLM3\",\"system_prompt\":\"\",\"roles\":[\"user\",\"assistant\"],\"intra_message_sep\":\"\",\"inter_message_sep\":\"\",\"stop\":[\"<|endoftext|>\",\"<|user|>\",\"<|observation|>\"],\"stop_token_ids\":[151329,151336,151338]}}","persist":true}'
|
||||
```
|
||||
|
||||
- 编辑注册embedding脚本
|
||||
```shell
|
||||
$ vim model_registrations_emb.sh
|
||||
# 添加以下内容。模型路径需要根据实际情况修改
|
||||
curl 'http://127.0.0.1:9997/v1/model_registrations/embedding' \
|
||||
-H 'Accept: */*' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Cookie: token=no_auth' \
|
||||
-H 'Origin: http://127.0.0.1:9997' \
|
||||
-H 'Referer: http://127.0.0.1:9997/ui/' \
|
||||
-H 'Sec-Fetch-Dest: empty' \
|
||||
-H 'Sec-Fetch-Mode: cors' \
|
||||
-H 'Sec-Fetch-Site: same-origin' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \
|
||||
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
|
||||
-H 'sec-ch-ua-mobile: ?0' \
|
||||
-H 'sec-ch-ua-platform: "Linux"' \
|
||||
--data-raw '{"model":"{\"model_name\":\"autodl-tmp-bge-large-zh\",\"dimensions\":768,\"max_tokens\":512,\"model_uri\":\"/root/model/bge-large-zh\",\"language\":[\"en\",\"zh\"]}","persist":true}'
|
||||
```
|
||||
|
||||
- 编辑启动模型脚本
|
||||
```shell
|
||||
$ vim start_models.sh
|
||||
# 添加以下内容。模型路径需要根据实际情况修改
|
||||
curl 'http://127.0.0.1:9997/v1/models' \
|
||||
-H 'Accept: */*' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Cookie: token=no_auth' \
|
||||
-H 'Origin: http://127.0.0.1:9997' \
|
||||
-H 'Referer: http://127.0.0.1:9997/ui/' \
|
||||
-H 'Sec-Fetch-Dest: empty' \
|
||||
-H 'Sec-Fetch-Mode: cors' \
|
||||
-H 'Sec-Fetch-Site: same-origin' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \
|
||||
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
|
||||
-H 'sec-ch-ua-mobile: ?0' \
|
||||
-H 'sec-ch-ua-platform: "Linux"' \
|
||||
--data-raw '{"model_uid":null,"model_name":"autodl-tmp-glm-4-9b-chat","model_type":"LLM","model_engine":"Transformers","model_format":"pytorch","model_size_in_billions":9,"quantization":"none","n_gpu":"auto","replica":1,"request_limits":null,"worker_ip":null,"gpu_idx":null}'
|
||||
```
|
||||
- 编辑启动embedding脚本
|
||||
```shell
|
||||
$ vim start_models_emb.sh
|
||||
# 添加以下内容。模型路径需要根据实际情况修改
|
||||
curl 'http://127.0.0.1:9997/v1/models' \
|
||||
-H 'Accept: */*' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Cookie: token=no_auth' \
|
||||
-H 'Origin: http://127.0.0.1:9997' \
|
||||
-H 'Referer: http://127.0.0.1:9997/ui/' \
|
||||
-H 'Sec-Fetch-Dest: empty' \
|
||||
-H 'Sec-Fetch-Mode: cors' \
|
||||
-H 'Sec-Fetch-Site: same-origin' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \
|
||||
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
|
||||
-H 'sec-ch-ua-mobile: ?0' \
|
||||
-H 'sec-ch-ua-platform: "Linux"' \
|
||||
--data-raw '{"model_uid":"bge-large-zh-v1.5","model_name":"autodl-tmp-bge-large-zh","model_type":"embedding","replica":1,"n_gpu":"auto","worker_ip":null,"gpu_idx":null}'
|
||||
```
|
||||
|
||||
- 启动模型
|
||||
```shell
|
||||
$ bash ./model_registrations.sh
|
||||
$ bash ./model_registrations_emb.sh
|
||||
$ bash ./start_models.sh
|
||||
$ bash ./start_models_emb.sh
|
||||
|
||||
```
|
||||
- 初始化chatchat配置
|
||||
```shell
|
||||
$ chatchat-config basic --verbose true
|
||||
$ chatchat-config basic --data ~/chatchat-data
|
||||
$ cp ~/miniconda3/envs/chatchat/lib/python3.8/site-packages/chatchat/configs/model_providers.yaml ~/chatchat-data/
|
||||
```
|
||||
|
||||
-- 设置模型
|
||||
```shell
|
||||
$ vim ~/chatchat-data/model_providers.yaml
|
||||
# 修改model_providers.yaml文件，添加如下内容
|
||||
|
||||
xinference:
|
||||
model_credential:
|
||||
- model: 'autodl-tmp-glm-4-9b-chat'
|
||||
model_type: 'llm'
|
||||
model_credentials:
|
||||
server_url: 'http://127.0.0.1:9997/'
|
||||
model_uid: 'autodl-tmp-glm-4-9b-chat'
|
||||
completion_type: 'chat'
|
||||
- model: 'bge-large-zh-v1.5'
|
||||
model_type: 'embeddings'
|
||||
model_credentials:
|
||||
server_url: 'http://127.0.0.1:9997/'
|
||||
model_uid: 'bge-large-zh-v1.5'
|
||||
```
|
||||
|
||||
-- 启动chatchat
|
||||
```shell
|
||||
$ chatchat -a
|
||||
|
||||
```
|
||||
Loading…
x
Reference in New Issue
Block a user