From 23a52259de98b255a6243b4cfa0f79cf694d1efe Mon Sep 17 00:00:00 2001 From: root <403644786@qq.com> Date: Mon, 15 Jul 2024 14:48:53 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86bnb=E9=87=8F?= =?UTF-8?q?=E5=8C=96=E7=9A=84=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- quantize/awq_quantize.py | 8 ++++---- quantize/gptq_quantize.py | 4 ++-- quantize/quantize_eval.py | 29 +++++++++++++++++++++++------ quantize/quantize_eval.sh | 10 +++++----- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/quantize/awq_quantize.py b/quantize/awq_quantize.py index b40b14e..1554b6c 100644 --- a/quantize/awq_quantize.py +++ b/quantize/awq_quantize.py @@ -5,12 +5,12 @@ from awq import AutoAWQForCausalLM from transformers import AutoTokenizer import os -model_path = '/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16' # model_path or model_id -quant_path = '/root/ld/ld_project/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path -quant_data_path='/root/ld/ld_project/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址 +model_path = '/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16' # model_path or model_id +quant_path = '/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path +quant_data_path='/root/ld/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址 quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } # "w_bit":4 or 8 quant_samples=512 # how many samples to use for calibration -custom_data=[{'question':'你叫什么名字。','answer':'我是openmbmb开源的小钢炮minicpm。'}, # 自定义数据集可用 +custom_data=[{'question':'鼻炎犯了怎么办','answer':'可以使用生理盐水进行清洗。'}, # 自定义数据集可用 {'question':'你有什么特色。','answer':'我很小，但是我很强。'}] # Load model model = AutoAWQForCausalLM.from_pretrained(model_path) diff --git a/quantize/gptq_quantize.py b/quantize/gptq_quantize.py index a98285b..78e10a4 100644 --- a/quantize/gptq_quantize.py +++ b/quantize/gptq_quantize.py @@ -98,8 +98,8 @@ def load_data(data_path, tokenizer, n_samples): def main(): parser = ArgumentParser() - parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16') - parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/ld_project/AutoGPTQ/examples/quantization/minicpm_1b_4bit') + parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16') + parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit') parser.add_argument("--bits", type=int, default=4, choices=[2, 3, 4])#do not use 8 bit parser.add_argument( "--group_size", diff --git a/quantize/quantize_eval.py b/quantize/quantize_eval.py index 82d350c..220d807 100644 --- a/quantize/quantize_eval.py +++ b/quantize/quantize_eval.py @@ -2,7 +2,7 @@ import torch import torch.nn as nn from tqdm import tqdm from datasets import load_dataset -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoModelForCausalLM, AutoTokenizer,AutoConfig import GPUtil import argparse @@ -13,6 +13,12 @@ parser.add_argument( default='', help="未量化前的模型路径。" ) +parser.add_argument( + "--bnb_path", + type=str, + default='', + help="bnb量化后的模型路径。" +) parser.add_argument( "--awq_path", type=str, @@ -83,9 +89,9 @@ if __name__ == "__main__": args = parser.parse_args() if args.model_path != "": + print("pretrained model：",args.model_path.split('/')[-1]) model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16, device_map='cuda', trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(args.model_path) - print("pretrained model：",args.model_path.split('/')[-1]) gpu_usage = GPUtil.getGPUs()[0].memoryUsed print(f"gpu usage: {round(gpu_usage/1024,2)}GB") evaluate_perplexity(model, tokenizer, args.data_path) @@ -93,10 +99,9 @@ if __name__ == "__main__": if args.awq_path != "": from awq import AutoAWQForCausalLM - + print("awq model：",args.awq_path.split('/')[-1]) model = AutoAWQForCausalLM.from_quantized(args.awq_path, fuse_layers=True,device_map={"":'cuda:0'}) tokenizer = AutoTokenizer.from_pretrained(args.awq_path) - print("awq model：",args.awq_path.split('/')[-1]) gpu_usage = GPUtil.getGPUs()[0].memoryUsed print(f"gpu usage: {round(gpu_usage/1024,2)}GB") evaluate_perplexity(model, tokenizer, args.data_path) @@ -105,11 +110,23 @@ if __name__ == "__main__": #we will support the autogptq later if args.gptq_path != "": from auto_gptq import AutoGPTQForCausalLM - + print("gptq model：",args.gptq_path.split('/')[-1]) tokenizer = AutoTokenizer.from_pretrained(args.gptq_path, use_fast=True) model = AutoGPTQForCausalLM.from_quantized(args.gptq_path, device="cuda:0",trust_remote_code=True) - print("gptq model：",args.gptq_path.split('/')[-1]) gpu_usage = GPUtil.getGPUs()[0].memoryUsed print(f"gpu usage: {round(gpu_usage/1024,2)}GB") evaluate_perplexity(model, tokenizer, args.data_path) + del model + if args.bnb_path != "": + from accelerate.utils import BnbQuantizationConfig + bnb_quantization_config = BnbQuantizationConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4") + print("bnb model：",args.gptq_path.split('/')[-1]) + # config=AutoConfig.from_pretrained(args.bnb_path,trust_remote_code=True) + # bnb_config=config.quantization_config + tokenizer = AutoTokenizer.from_pretrained(args.bnb_path, use_fast=True) + model = AutoModelForCausalLM.from_pretrained(args.bnb_path, trust_remote_code=True,)#quantization_config=bnb_config,) + gpu_usage = GPUtil.getGPUs()[0].memoryUsed + print(f"gpu usage: {round(gpu_usage/1024,2)}GB") + evaluate_perplexity(model, tokenizer, args.data_path) + del model diff --git a/quantize/quantize_eval.sh b/quantize/quantize_eval.sh index 4002303..e080659 100644 --- a/quantize/quantize_eval.sh +++ b/quantize/quantize_eval.sh @@ -1,8 +1,8 @@ #!/bin/bash -awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit" -gptq_path="" -model_path="" - +awq_path="/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit" +gptq_path="/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit" +model_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" +bnb_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" python quantize_eval.py --awq_path "${awq_path}" \ - --model_path "${model_path}" --gptq_path "${gptq_path}" \ No newline at end of file + --model_path "${model_path}" --gptq_path "${gptq_path}" --bnb_path "${bnb_path}" \ No newline at end of file From 08514cd9ec4628a9d7bd457ad1a7c920ac33e512 Mon Sep 17 00:00:00 2001 From: root <403644786@qq.com> Date: Mon, 15 Jul 2024 14:49:32 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86bnb=E7=9A=84?= =?UTF-8?q?=E9=87=8F=E5=8C=96demo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- quantize/bnb_quantize.py | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 quantize/bnb_quantize.py diff --git a/quantize/bnb_quantize.py b/quantize/bnb_quantize.py new file mode 100644 index 0000000..c32cea6 --- /dev/null +++ b/quantize/bnb_quantize.py @@ -0,0 +1,57 @@ +""" +the script will use bitandbytes to quantize the MiniCPM language model. +the be quantized model can be finetuned by MiniCPM or not. +you only need to set the model_path 、save_path and run bash code + +cd MiniCPM +python quantize/bnb_quantize.py + +you will get the quantized model in save_path、quantized_model test time and gpu usage +""" + + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig +import time +import torch +import GPUtil +import os + +model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型下载地址 +save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址 +device = "cuda" if torch.cuda.is_available() else "cpu" + +# 创建一个配置对象来指定量化参数 +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, # 是否进行4bit量化 + load_in_8bit=False, # 是否进行8bit量化 + bnb_4bit_compute_dtype=torch.float16, # 计算精度设置 + bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式 + bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4 + bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化 + llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32 + llm_int8_has_fp16_weight=False, # 是否启用混合精度 + #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块 + llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化 +) + +tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_path, + device_map=device, # 分配模型到device + quantization_config=quantization_config, + trust_remote_code=True, +) + +gpu_usage = GPUtil.getGPUs()[0].memoryUsed +start = time.time() +response = model.chat(tokenizer, "<用户>给我讲一个故事",history=[], temperature=0.5, top_p=0.8, repetition_penalty=1.02) # 模型推理 +print("量化后输出", response) +print("量化后推理用时", time.time() - start) +print(f"量化后显存占用: {round(gpu_usage/1024,2)}GB") + + +# 保存模型和分词器 +os.makedirs(save_path, exist_ok=True) +model.save_pretrained(save_path, safe_serialization=True) +tokenizer.save_pretrained(save_path) From 4a7761df1fb23ded50f4c126a8b6968ff30230fc Mon Sep 17 00:00:00 2001 From: root <403644786@qq.com> Date: Mon, 15 Jul 2024 14:55:31 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86bnb=E9=87=8F?= =?UTF-8?q?=E5=8C=96=E6=A8=A1=E5=9E=8B=E7=9A=84readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- quantize/readme.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/quantize/readme.md b/quantize/readme.md index 344b458..17f93fa 100644 --- a/quantize/readme.md +++ b/quantize/readme.md @@ -45,15 +45,45 @@ ``` 5. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。 + +

+ +**bnb量化** +1. 在quantize/bnb_quantize.py 文件中修改根据注释修改配置参数： +```python +model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型地址 +save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址 +``` +2. 更多量化参数可根据注释以及llm.int8()算法进行修改： +```python +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, # 是否进行4bit量化 + load_in_8bit=False, # 是否进行8bit量化 + bnb_4bit_compute_dtype=torch.float16, # 计算精度设置 + bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式 + bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4 + bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化 + llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32 + llm_int8_has_fp16_weight=False, # 是否启用混合精度 + #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块 + llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化 +) +``` +3. 运行quantize/bnb_quantize.py文件,在设置的save_path目录下可得bnb量化后的模型。 +```python +cd MiniCPM/quantize +python bnb_quantize.py +```

**量化测试** 1. 命令行进入到 MiniCPM/quantize 目录下 -2. 修改quantize_eval.sh文件中awq_path，gptq_path，awq_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型： +2. 修改quantize_eval.sh文件中awq_path,gptq_path,awq_path,bnb_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型： ``` awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit" gptq_path="" model_path="" + bnb_path="" ``` 3. 在MiniCPM/quantize路径下命令行输入： ``` From 3d18712792aaa4a4bbeda350aaaa7b645389d185 Mon Sep 17 00:00:00 2001 From: root <403644786@qq.com> Date: Mon, 15 Jul 2024 15:02:22 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86bnb=E9=87=8F?= =?UTF-8?q?=E5=8C=96=E7=9A=84=E5=BF=AB=E9=80=9F=E5=AF=BC=E8=88=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 34c122c..7502eed 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ MiniCPM 是面壁智能与清华大学自然语言处理实验室共同开源的 |-------------|------------|-----------|-----------| |[Transformers](#Huggingface模型)|[Transformers](#transformer_finetune)|[MLC部署](#MLC)|[GPTQ](#gptq)| |[vLLM](#vllm-推理)|[mlx_finetune](#mlx)|[llama.cpp](#llama.cpp)|[AWQ](#awq)| -|[llama.cpp](#llama.cpp)|[llama_factory](./finetune/llama_factory_example/README.md)||[困惑度测试](#quantize_test)| -|[ollama](#ollama)|||| +|[llama.cpp](#llama.cpp)|[llama_factory](./finetune/llama_factory_example/README.md)||[bnb](#bnb)| +|[ollama](#ollama)|||[量化测试](#quantize_test)| |[fastllm](#fastllm)|||| |[mlx_lm](#mlx_lm)|||| |[powerinfer](#powerinfer)|||| @@ -379,6 +379,35 @@ cd PowerInfer 5. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。

+

+ +**bnb量化** +1. 在quantize/bnb_quantize.py 文件中修改根据注释修改配置参数： +```python +model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16" # 模型地址 +save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4" # 量化模型保存地址 +``` +2. 更多量化参数可根据注释以及llm.int8()算法进行修改(optional)： +```python +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, # 是否进行4bit量化 + load_in_8bit=False, # 是否进行8bit量化 + bnb_4bit_compute_dtype=torch.float16, # 计算精度设置 + bnb_4bit_quant_storage=torch.uint8, # 量化权重的储存格式 + bnb_4bit_quant_type="nf4", # 量化格式，这里用的是正太分布的int4 + bnb_4bit_use_double_quant=True, # 是否采用双量化，即对zeropoint和scaling参数进行量化 + llm_int8_enable_fp32_cpu_offload=False, # 是否llm使用int8，cpu上保存的参数使用fp32 + llm_int8_has_fp16_weight=False, # 是否启用混合精度 + #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"], # 不进行量化的模块 + llm_int8_threshold=6.0, # llm.int8()算法中的离群值，根据这个值区分是否进行量化 +) +``` +3. 运行quantize/bnb_quantize.py文件,在设置的save_path目录下可得bnb量化后的模型。 +```python +cd MiniCPM/quantize +python bnb_quantize.py +``` + **量化测试** 1. 命令行进入到 MiniCPM/quantize 目录下 2. 修改quantize_eval.sh文件中awq_path，gptq_path，awq_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：