mirror of
https://github.com/RYDE-WORK/MiniCPM.git
synced 2026-01-19 21:03:39 +08:00
67 lines
1.8 KiB
Plaintext
67 lines
1.8 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 1. 准备数据集\n",
|
|
"\n",
|
|
"将数据集转换为更通用的格式\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# 转换为 ChatML 格式\n",
|
|
"import os\n",
|
|
"import shutil\n",
|
|
"import json\n",
|
|
"\n",
|
|
"input_dir = \"data/AdvertiseGen\"\n",
|
|
"output_dir = \"data/mlx_AdvertiseGen\"\n",
|
|
"if os.path.exists(output_dir):\n",
|
|
" shutil.rmtree(output_dir)\n",
|
|
"os.makedirs(output_dir, exist_ok=True)\n",
|
|
"\n",
|
|
"for fn in [\"train.json\", \"dev.json\"]:\n",
|
|
" data_out_list = []\n",
|
|
" with open(os.path.join(input_dir, fn), \"r\") as f, open(os.path.join(output_dir, fn), \"w\") as fo:\n",
|
|
" for line in f:\n",
|
|
" if len(line.strip()) > 0:\n",
|
|
" data = json.loads(line)\n",
|
|
" data_out = {\"input\":data['content'],'prompt':\"/n请为以下关键词生成一条广告语。\",'output':data['summary']}\n",
|
|
" data_out_list.append(data_out)\n",
|
|
"\n",
|
|
" for d in data_out_list:\n",
|
|
" json_str = json.dumps(d,ensure_ascii=False) # 将字典转换为JSON字符串\n",
|
|
" fo.write(json_str + '\\n') # 写入字符串并添加换行符\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "base",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|