增加了bnb量化的测试

quantize/awq_quantize.py

@@ -5,12 +5,12 @@ from awq import AutoAWQForCausalLM
 from transformers import AutoTokenizer
 import os
 
-model_path = '/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16' # model_path or model_id
+model_path = '/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16' # model_path or model_id
-quant_path = '/root/ld/ld_project/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
+quant_path = '/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
-quant_data_path='/root/ld/ld_project/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址
+quant_data_path='/root/ld/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带数据集地址
 quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } # "w_bit":4 or 8
 quant_samples=512 # how many samples to use for calibration
-custom_data=[{'question':'你叫什么名字。','answer':'我是openmbmb开源的小钢炮minicpm。'}, # 自定义数据集可用
+custom_data=[{'question':'鼻炎犯了怎么办','answer':'可以使用生理盐水进行清洗。'}, # 自定义数据集可用
                  {'question':'你有什么特色。','answer':'我很小，但是我很强。'}]
 # Load model
 model = AutoAWQForCausalLM.from_pretrained(model_path)

quantize/gptq_quantize.py

@@ -98,8 +98,8 @@ def load_data(data_path, tokenizer, n_samples):
 
 def main():
     parser = ArgumentParser()
-    parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16')
+    parser.add_argument("--pretrained_model_dir", type=str,default='/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16')
-    parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/ld_project/AutoGPTQ/examples/quantization/minicpm_1b_4bit')
+    parser.add_argument("--quantized_model_dir", type=str, default='/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit')
     parser.add_argument("--bits", type=int, default=4, choices=[2, 3, 4])#do not use 8 bit
     parser.add_argument(
         "--group_size",

quantize/quantize_eval.py

@@ -2,7 +2,7 @@ import torch
 import torch.nn as nn
 from tqdm import tqdm
 from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer,AutoConfig
 import GPUtil
 import argparse
 
@@ -13,6 +13,12 @@ parser.add_argument(
     default='',  
     help="未量化前的模型路径。"  
 )
+parser.add_argument(
+    "--bnb_path",  
+    type=str,  
+    default='',  
+    help="bnb量化后的模型路径。"  
+)
 parser.add_argument(
     "--awq_path",  
     type=str,  
@@ -83,9 +89,9 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     if args.model_path != "":
+        print("pretrained model：",args.model_path.split('/')[-1])
         model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16, device_map='cuda', trust_remote_code=True)
         tokenizer = AutoTokenizer.from_pretrained(args.model_path)
-        print("pretrained model：",args.model_path.split('/')[-1])
         gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
         print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
         evaluate_perplexity(model, tokenizer, args.data_path)
@@ -93,10 +99,9 @@ if __name__ == "__main__":
 
     if args.awq_path != "":
         from awq import AutoAWQForCausalLM
-
+        print("awq model：",args.awq_path.split('/')[-1])
         model = AutoAWQForCausalLM.from_quantized(args.awq_path, fuse_layers=True,device_map={"":'cuda:0'})
         tokenizer = AutoTokenizer.from_pretrained(args.awq_path)
-        print("awq model：",args.awq_path.split('/')[-1])
         gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
         print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
         evaluate_perplexity(model, tokenizer, args.data_path)
@@ -105,11 +110,23 @@ if __name__ == "__main__":
 #we will support the autogptq  later
     if args.gptq_path != "":
         from auto_gptq import AutoGPTQForCausalLM
-
+        print("gptq model：",args.gptq_path.split('/')[-1])
         tokenizer = AutoTokenizer.from_pretrained(args.gptq_path, use_fast=True)
         model = AutoGPTQForCausalLM.from_quantized(args.gptq_path, device="cuda:0",trust_remote_code=True)
-        print("gptq model：",args.gptq_path.split('/')[-1])
         gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
         print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
         evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
     
+    if args.bnb_path != "":
+        from accelerate.utils import BnbQuantizationConfig
+        bnb_quantization_config = BnbQuantizationConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
+        print("bnb model：",args.gptq_path.split('/')[-1])
+        # config=AutoConfig.from_pretrained(args.bnb_path,trust_remote_code=True)
+        # bnb_config=config.quantization_config
+        tokenizer = AutoTokenizer.from_pretrained(args.bnb_path, use_fast=True)
+        model = AutoModelForCausalLM.from_pretrained(args.bnb_path, trust_remote_code=True,)#quantization_config=bnb_config,)
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model

quantize/quantize_eval.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
+awq_path="/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit"
-gptq_path=""
+gptq_path="/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit"
-model_path=""
+model_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16"
-
+bnb_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4"
 python quantize_eval.py --awq_path "${awq_path}" \
- --model_path "${model_path}" --gptq_path "${gptq_path}"
+ --model_path "${model_path}" --gptq_path "${gptq_path}" --bnb_path "${bnb_path}"