diff --git a/README.md b/README.md
index 34c122c..7502eed 100644
--- a/README.md
+++ b/README.md
@@ -64,8 +64,8 @@ MiniCPM 是面壁智能与清华大学自然语言处理实验室共同开源的
|-------------|------------|-----------|-----------|
|[Transformers](#Huggingface模型)|[Transformers](#transformer_finetune)|[MLC部署](#MLC)|[GPTQ](#gptq)|
|[vLLM](#vllm-推理)|[mlx_finetune](#mlx)|[llama.cpp](#llama.cpp)|[AWQ](#awq)|
-|[llama.cpp](#llama.cpp)|[llama_factory](./finetune/llama_factory_example/README.md)||[困惑度测试](#quantize_test)|
-|[ollama](#ollama)||||
+|[llama.cpp](#llama.cpp)|[llama_factory](./finetune/llama_factory_example/README.md)||[bnb](#bnb)|
+|[ollama](#ollama)|||[量化测试](#quantize_test)|
|[fastllm](#fastllm)||||
|[mlx_lm](#mlx_lm)||||
|[powerinfer](#powerinfer)||||
@@ -379,6 +379,35 @@ cd PowerInfer
5. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。
+
+
+**bnb量化**
+1. 在quantize/bnb_quantize.py 文件中修改根据注释修改配置参数：
+```python
+model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型地址
+save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址
+```
+2. 更多量化参数可根据注释以及llm.int8()算法进行修改(optional)：
+```python
+quantization_config = BitsAndBytesConfig(
+ load_in_4bit=True, # 是否进行4bit量化
+ load_in_8bit=False, # 是否进行8bit量化
+ bnb_4bit_compute_dtype=torch.float16, # 计算精度设置
+ bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式
+ bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4
+ bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化
+ llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32
+ llm_int8_has_fp16_weight=False, # 是否启用混合精度
+ #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块
+ llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化
+)
+```
+3. 运行quantize/bnb_quantize.py文件,在设置的save_path目录下可得bnb量化后的模型。
+```python
+cd MiniCPM/quantize
+python bnb_quantize.py
+```
+
**量化测试**
1. 命令行进入到 MiniCPM/quantize 目录下
2. 修改quantize_eval.sh文件中awq_path，gptq_path，awq_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：
diff --git a/quantize/awq_quantize.py b/quantize/awq_quantize.py
index b40b14e..1554b6c 100644
--- a/quantize/awq_quantize.py
+++ b/quantize/awq_quantize.py
@@ -5,12 +5,12 @@ from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer
import os
-model_path = '/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16' # model_path or model_id
-quant_path = '/root/ld/ld_project/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
-quant_data_path='/root/ld/ld_project/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址
+model_path = '/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16' # model_path or model_id
+quant_path = '/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
+quant_data_path='/root/ld/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址
quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } # "w_bit":4 or 8
quant_samples=512 # how many samples to use for calibration
-custom_data=[{'question':'你叫什么名字。','answer':'我是openmbmb开源的小钢炮minicpm。'}, # 自定义数据集可用
+custom_data=[{'question':'鼻炎犯了怎么办','answer':'可以使用生理盐水进行清洗。'}, # 自定义数据集可用
{'question':'你有什么特色。','answer':'我很小，但是我很强。'}]
# Load model
model = AutoAWQForCausalLM.from_pretrained(model_path)
diff --git a/quantize/bnb_quantize.py b/quantize/bnb_quantize.py
new file mode 100644
index 0000000..c32cea6
--- /dev/null
+++ b/quantize/bnb_quantize.py
@@ -0,0 +1,57 @@
+"""
+the script will use bitandbytes to quantize the MiniCPM language model.
+the be quantized model can be finetuned by MiniCPM or not.
+you only need to set the model_path 、save_path and run bash code
+
+cd MiniCPM
+python quantize/bnb_quantize.py
+
+you will get the quantized model in save_path、quantized_model test time and gpu usage
+"""
+
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import time
+import torch
+import GPUtil
+import os
+
+model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型下载地址
+save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# 创建一个配置对象来指定量化参数
+quantization_config = BitsAndBytesConfig(
+ load_in_4bit=True, # 是否进行4bit量化
+ load_in_8bit=False, # 是否进行8bit量化
+ bnb_4bit_compute_dtype=torch.float16, # 计算精度设置
+ bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式
+ bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4
+ bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化
+ llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32
+ llm_int8_has_fp16_weight=False, # 是否启用混合精度
+ #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块
+ llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+ model_path,
+ device_map=device, # 分配模型到device
+ quantization_config=quantization_config,
+ trust_remote_code=True,
+)
+
+gpu_usage = GPUtil.getGPUs()[0].memoryUsed
+start = time.time()
+response = model.chat(tokenizer, "<用户>给我讲一个故事",history=[], temperature=0.5, top_p=0.8, repetition_penalty=1.02) # 模型推理
+print("量化后输出", response)
+print("量化后推理用时", time.time() - start)
+print(f"量化后显存占用: {round(gpu_usage/1024,2)}GB")
+
+
+# 保存模型和分词器
+os.makedirs(save_path, exist_ok=True)
+model.save_pretrained(save_path, safe_serialization=True)
+tokenizer.save_pretrained(save_path)
diff --git a/quantize/gptq_quantize.py b/quantize/gptq_quantize.py
index a98285b..78e10a4 100644
--- a/quantize/gptq_quantize.py
+++ b/quantize/gptq_quantize.py
@@ -98,8 +98,8 @@ def load_data(data_path, tokenizer, n_samples):
def main():
parser = ArgumentParser()
- parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16')
- parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/ld_project/AutoGPTQ/examples/quantization/minicpm_1b_4bit')
+ parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16')
+ parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit')
parser.add_argument("--bits", type=int, default=4, choices=[2, 3, 4])#do not use 8 bit
parser.add_argument(
"--group_size",
diff --git a/quantize/quantize_eval.py b/quantize/quantize_eval.py
index 82d350c..220d807 100644
--- a/quantize/quantize_eval.py
+++ b/quantize/quantize_eval.py
@@ -2,7 +2,7 @@ import torch
import torch.nn as nn
from tqdm import tqdm
from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer,AutoConfig
import GPUtil
import argparse
@@ -13,6 +13,12 @@ parser.add_argument(
default='',
help="未量化前的模型路径。"
)
+parser.add_argument(
+ "--bnb_path",
+ type=str,
+ default='',
+ help="bnb量化后的模型路径。"
+)
parser.add_argument(
"--awq_path",
type=str,
@@ -83,9 +89,9 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.model_path != "":
+ print("pretrained model：",args.model_path.split('/')[-1])
model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16, device_map='cuda', trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(args.model_path)
- print("pretrained model：",args.model_path.split('/')[-1])
gpu_usage = GPUtil.getGPUs()[0].memoryUsed
print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
evaluate_perplexity(model, tokenizer, args.data_path)
@@ -93,10 +99,9 @@ if __name__ == "__main__":
if args.awq_path != "":
from awq import AutoAWQForCausalLM
-
+ print("awq model：",args.awq_path.split('/')[-1])
model = AutoAWQForCausalLM.from_quantized(args.awq_path, fuse_layers=True,device_map={"":'cuda:0'})
tokenizer = AutoTokenizer.from_pretrained(args.awq_path)
- print("awq model：",args.awq_path.split('/')[-1])
gpu_usage = GPUtil.getGPUs()[0].memoryUsed
print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
evaluate_perplexity(model, tokenizer, args.data_path)
@@ -105,11 +110,23 @@ if __name__ == "__main__":
#we will support the autogptq later
if args.gptq_path != "":
from auto_gptq import AutoGPTQForCausalLM
-
+ print("gptq model：",args.gptq_path.split('/')[-1])
tokenizer = AutoTokenizer.from_pretrained(args.gptq_path, use_fast=True)
model = AutoGPTQForCausalLM.from_quantized(args.gptq_path, device="cuda:0",trust_remote_code=True)
- print("gptq model：",args.gptq_path.split('/')[-1])
gpu_usage = GPUtil.getGPUs()[0].memoryUsed
print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
evaluate_perplexity(model, tokenizer, args.data_path)
+ del model
+ if args.bnb_path != "":
+ from accelerate.utils import BnbQuantizationConfig
+ bnb_quantization_config = BnbQuantizationConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
+ print("bnb model：",args.gptq_path.split('/')[-1])
+ # config=AutoConfig.from_pretrained(args.bnb_path,trust_remote_code=True)
+ # bnb_config=config.quantization_config
+ tokenizer = AutoTokenizer.from_pretrained(args.bnb_path, use_fast=True)
+ model = AutoModelForCausalLM.from_pretrained(args.bnb_path, trust_remote_code=True,)#quantization_config=bnb_config,)
+ gpu_usage = GPUtil.getGPUs()[0].memoryUsed
+ print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+ evaluate_perplexity(model, tokenizer, args.data_path)
+ del model
diff --git a/quantize/quantize_eval.sh b/quantize/quantize_eval.sh
index 4002303..e080659 100644
--- a/quantize/quantize_eval.sh
+++ b/quantize/quantize_eval.sh
@@ -1,8 +1,8 @@
#!/bin/bash
-awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
-gptq_path=""
-model_path=""
-
+awq_path="/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit"
+gptq_path="/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit"
+model_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16"
+bnb_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4"
python quantize_eval.py --awq_path "${awq_path}" \
- --model_path "${model_path}" --gptq_path "${gptq_path}"
\ No newline at end of file
+ --model_path "${model_path}" --gptq_path "${gptq_path}" --bnb_path "${bnb_path}"
\ No newline at end of file
diff --git a/quantize/readme.md b/quantize/readme.md
index 344b458..17f93fa 100644
--- a/quantize/readme.md
+++ b/quantize/readme.md
@@ -45,15 +45,45 @@
```
5. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。
+
+
+
+**bnb量化**
+1. 在quantize/bnb_quantize.py 文件中修改根据注释修改配置参数：
+```python
+model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型地址
+save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址
+```
+2. 更多量化参数可根据注释以及llm.int8()算法进行修改：
+```python
+quantization_config = BitsAndBytesConfig(
+ load_in_4bit=True, # 是否进行4bit量化
+ load_in_8bit=False, # 是否进行8bit量化
+ bnb_4bit_compute_dtype=torch.float16, # 计算精度设置
+ bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式
+ bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4
+ bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化
+ llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32
+ llm_int8_has_fp16_weight=False, # 是否启用混合精度
+ #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块
+ llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化
+)
+```
+3. 运行quantize/bnb_quantize.py文件,在设置的save_path目录下可得bnb量化后的模型。
+```python
+cd MiniCPM/quantize
+python bnb_quantize.py
+```
**量化测试**
1. 命令行进入到 MiniCPM/quantize 目录下
-2. 修改quantize_eval.sh文件中awq_path，gptq_path，awq_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：
+2. 修改quantize_eval.sh文件中awq_path,gptq_path,awq_path,bnb_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：
```
awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
gptq_path=""
model_path=""
+ bnb_path=""
```
3. 在MiniCPM/quantize路径下命令行输入：
```