Merge pull request #157 from LDLINGLINGLING/main

add support autoawq for minicpm

README.md

@@ -49,6 +49,7 @@ MiniCPM 是面壁智能与清华大学自然语言处理实验室共同开源的
 - [更新日志](#0)
 - [模型下载](#1)
 - [快速上手](#2)
+- [模型量化](#quantize)
 - [开源社区](#community)
 - [评测结果](#3)
 - [手机部署](#4)
@@ -258,7 +259,9 @@ print(model.response("<用户>山东省最高的山是哪座山, 它比黄山高
     ```shell
     python -m mlx_lm.generate --model mlx-community/MiniCPM-2B-sft-bf16-llama-format-mlx --prompt "hello, tell me a joke." --trust-remote-code
     ```
+<p id="community"></p>
 
+## 模型量化
 **gptq量化**
 1. 首先git获取[minicpm_gptqd代码](https://github.com/LDLINGLINGLING/AutoGPTQ/tree/minicpm_gptq)
 2. 进入minicpm_gptqd主目录./AutoGPTQ，命令行输入：
@@ -271,6 +274,29 @@ print(model.response("<用户>山东省最高的山是哪座山, 它比黄山高
     python quant_with_alpaca.py --pretrained_model_dir no_quantized_path --quantized_model_dir save_path --bits 4
     ```
 5. 可以使用./AutoGPTQ/examples/quantization/inference.py进行推理，也可以参考前文使用vllm对量化后的模型，单卡4090下minicpm-1b-int4模型vllm推理在2000token/s左右。
+
+**awq量化**
+1. 在quantize/awq_quantize.py 文件中修改根据注释修改配置参数：model_path , quant_path, quant_data_path , quant_config, quant_samples, 如需自定数据集则需要修改 custom_data。
+2. 在quantize/quantize_data文件下已经提供了alpaca和wiki_text两个数据集作为量化校准集，如果需要自定义数据集，修改quantize/awq_quantize.py中的custom_data变量，如：
+    ```
+    custom_data=[{'question':'过敏性鼻炎有什么症状？','answer':'过敏性鼻炎可能鼻塞，流鼻涕，头痛等症状反复发作，严重时建议及时就医。'},
+                 {'question':'1+1等于多少？','answer':'等于2'}]
+    ```
+3. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。
+
+**量化测试**
+1. 命令行进入到 MiniCPM/quantize 目录下
+2. 修改quantize_eval.sh文件中awq_path，gptq_path，awq_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：
+  ```
+    awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
+    gptq_path=""
+    model_path=""
+  ```
+3. 在MiniCPM/quantize路径下命令行输入：
+  ```
+    bash quantize_eval.sh
+  ```
+4. 窗口将输出该模型的内存占用情况、困惑度。
 <p id="community"></p>
 
 ## 开源社区

finetune/finetune.py

@@ -12,7 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, Trainer,
 
 @dataclass
 class ModelArguments:
-    model_name_or_path: Optional[str] = field(default="baichuan-inc/Baichuan2-7B-Base")
+    model_name_or_path: Optional[str] = field(default="openbmb/MiniCPM-2B-sft-bf16")
 
 
 @dataclass
@@ -195,4 +195,4 @@ if __name__ == "__main__":
 
     trainer.train()
     # save the incremental PEFT weights, more details can be found in https://huggingface.co/blog/peft
-    # model.save_pretrained("output_dir") 
+    # model.save_pretrained("output_dir") 

quantize/awq_quantize.py

@@ -0,0 +1,44 @@
+
+
+from datasets import load_dataset
+from awq import AutoAWQForCausalLM
+from transformers import AutoTokenizer
+import os
+
+model_path = '/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16' # model_path or model_id
+quant_path = '/root/ld/ld_project/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
+quant_data_path='/root/ld/ld_project/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带
+quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } # "w_bit":4 or 8
+quant_samples=512 # how many samples to use for calibration
+custom_data=[{'question':'你叫什么名字。','answer':'我是openmbmb开源的小钢炮minicpm。'},
+                 {'question':'你有什么特色。','answer':'我很小，但是我很强。'}]
+# Load model
+model = AutoAWQForCausalLM.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True,device_map={"": "cuda:0"})
+
+# Define data loading methods
+def load_alpaca(quant_data_path):
+    data = load_dataset(quant_data_path, split="train") # Set the absolute path to alpaca or huggingface id
+
+    # concatenate data
+    def concatenate_data(x):
+        return {"text": '<s><用户>'+x['instruction']  + x['input'] + '<AI>' + '\n' + x['output']}
+    
+    concatenated = data.map(concatenate_data)[:quant_samples]
+    return [text for text in concatenated["text"]]
+
+def load_wikitext(quant_data_path):
+    data = load_dataset(quant_data_path,  split="train")
+    return [text for text in data["text"] if text.strip() != '' and len(text.split(' ')) > 20][:quant_samples]
+
+def load_cust_data(custom_data):
+    quant_data=['<s><用户>'+i['question'] + '<AI>'  + i['answer'] + '<s>' for i in custom_data]
+    return quant_data[:quant_samples]
+# Quantize
+model.quantize(tokenizer, quant_config=quant_config, calib_data=load_wikitext(quant_data_path=quant_data_path))
+
+# Save quantized model
+model.save_quantized(quant_path)
+tokenizer.save_pretrained(quant_path)
+
+print(f'Model is quantized and saved at "{quant_path}"')

quantize/quantize_data/alpaca/downloads/69e71f4ebbd3d7f8f822d46b759228b3fb785d2c74d87f14126cf8da4f403610.json

@@ -0,0 +1 @@
+{"url": "hf://datasets/tatsu-lab/alpaca@dce01c9b08f87459cf36a430d809084718273017/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet", "etag": null}

quantize/quantize_data/alpaca/tatsu-lab___alpaca/default/0.0.0/dce01c9b08f87459cf36a430d809084718273017/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"instruction": {"dtype": "string", "_type": "Value"}, "input": {"dtype": "string", "_type": "Value"}, "output": {"dtype": "string", "_type": "Value"}, "text": {"dtype": "string", "_type": "Value"}}, "builder_name": "parquet", "dataset_name": "alpaca", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 46208623, "num_examples": 52002, "dataset_name": "alpaca"}}, "download_checksums": {"hf://datasets/tatsu-lab/alpaca@dce01c9b08f87459cf36a430d809084718273017/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet": {"num_bytes": 24246638, "checksum": null}}, "download_size": 24246638, "dataset_size": 46208623, "size_in_bytes": 70455261}

quantize/quantize_data/wikitext/downloads/300e51c2d3e98b6cf3739090e25ea220d1a568457b1b2d97857e1cf71da48434.json

@@ -0,0 +1 @@
+{"url": "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/validation-00000-of-00001.parquet", "etag": null}

quantize/quantize_data/wikitext/downloads/5e548f01091b2f01906ea7be35b8a1a6aa74842629dc550b7a02ca0821f1002d.json

@@ -0,0 +1 @@
+{"url": "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/test-00000-of-00001.parquet", "etag": null}

quantize/quantize_data/wikitext/downloads/e7f32d67ecf57307f5bc9834102f2f01ad03802eea0fdfc86172a9ec98676280.json

@@ -0,0 +1 @@
+{"url": "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/train-00000-of-00001.parquet", "etag": null}

quantize/quantize_data/wikitext/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"text": {"dtype": "string", "_type": "Value"}}, "builder_name": "parquet", "dataset_name": "wikitext", "config_name": "wikitext-2-raw-v1", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1305088, "num_examples": 4358, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 11061717, "num_examples": 36718, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1159288, "num_examples": 3760, "dataset_name": "wikitext"}}, "download_checksums": {"hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/test-00000-of-00001.parquet": {"num_bytes": 732610, "checksum": null}, "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/train-00000-of-00001.parquet": {"num_bytes": 6357543, "checksum": null}, "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/validation-00000-of-00001.parquet": {"num_bytes": 657209, "checksum": null}}, "download_size": 7747362, "dataset_size": 13526093, "size_in_bytes": 21273455}

quantize/quantize_eval.py

@@ -0,0 +1,115 @@
+import torch
+import torch.nn as nn
+from tqdm import tqdm
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import GPUtil
+import argparse
+
+parser = argparse.ArgumentParser(description="========量化困惑度测试========")
+parser.add_argument(
+    "--model_path",  
+    type=str,  
+    default='',  
+    help="未量化前的模型路径。"  
+)
+parser.add_argument(
+    "--awq_path",  
+    type=str,  
+    default='',  
+    help="awq量化后的模型保存路径。"  
+)
+#we will support gptq later
+parser.add_argument(
+    "--gptq_path",  
+    type=str,  
+    default='',  
+    help="gptq量化后的模型保存路径。"  
+)
+parser.add_argument(
+    "--data_path",  
+    type=str,  
+    default='quantize_data/wikitext',  
+    help="可以是以后的量化数据集，示例中默认为wiki_text"  
+)
+
+def get_device():
+    if torch.backends.mps.is_available():
+        return "mps"
+    elif torch.cuda.is_available():
+        return "cuda:0"
+    else:
+        return "cpu"
+
+def evaluate_perplexity(model, tokenizer,data_path):
+    def _perplexity(nlls, n_samples, seqlen):
+        return torch.exp(torch.stack(nlls).sum() / (n_samples * seqlen))
+
+    data = load_dataset(data_path,  split="test")
+    data = tokenizer("\n\n".join(data["text"]), return_tensors="pt")
+    data = data.input_ids.to('cuda:0')
+
+    seqlen = 2048
+    model = model.eval()
+    n_samples = data.numel() // seqlen
+
+    nlls = []
+
+    with tqdm(range(n_samples), desc="Perplexity -") as progress_bar:
+        for i in progress_bar:
+            start_index = i * seqlen
+            end_index = (i + 1) * seqlen
+            batch = data[:, start_index:end_index].to('cuda:0')
+            with torch.no_grad():
+                logits = model(batch).logits
+            shift_logits = logits[:, :-1, :].contiguous().float()
+            shift_labels = data[:, start_index:end_index][:, 1:]
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
+            neg_log_likelihood = loss.float() * seqlen
+            nlls.append(neg_log_likelihood)
+
+            curr_ppl = _perplexity(nlls, i + 1, seqlen)
+            progress_bar.set_description(f"Perplexity {curr_ppl:.3f}")
+
+    ppl = _perplexity(nlls, n_samples, seqlen)
+
+    return ppl.item()
+
+if __name__ == "__main__":
+    
+    args = parser.parse_args()
+
+    if args.model_path != "":
+        model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16, device_map='cuda', trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+        print("pretrained model：",args.model_path.split('/')[-1])
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
+
+    if args.awq_path != "":
+        from awq import AutoAWQForCausalLM
+
+        model = AutoAWQForCausalLM.from_quantized(args.awq_path, fuse_layers=True,device_map={"":'cuda:0'})
+        tokenizer = AutoTokenizer.from_pretrained(args.awq_path)
+        print("awq model：",args.awq_path.split('/')[-1])
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
+
+#we will support the autogptq  later
+    if args.gptq_path != "":
+        from auto_gptq import AutoGPTQForCausalLM
+
+        tokenizer = AutoTokenizer.from_pretrained(args.gptq_path, use_fast=True)
+        model = AutoGPTQForCausalLM.from_quantized(args.gptq_path, device="cuda:0",trust_remote_code=True)
+        print("gptq model：",args.gptq_path.split('/')[-1])
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+    

quantize/quantize_eval.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
+gptq_path=""
+model_path=""
+
+python quantize_eval.py --awq_path "${awq_path}" \
+ --model_path "${model_path}" --gptq_path "${gptq_path}"