RYDE-WORK/MiniCPM
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/MiniCPM.git synced 2026-01-19 12:53:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

使用encode方法，代码可读性更强

Browse Source
This commit is contained in:
root 2024-06-21 16:08:39 +08:00
parent 8ae10c60ff
commit 062ea5264a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
finetune/finetune.py
View File

@ -55,8 +55,8 @@ class SupervisedDataset(Dataset):
self.data = json.load(open(data_path))
self.tokenizer = tokenizer
self.model_max_length = model_max_length
self.user_tokens = self.tokenizer(user_tokens)['input_ids']#针对不同模型，都可以对应到<用户>的id
self.assistant_tokens = self.tokenizer(assistant_tokens)['input_ids']#针对不同模型，都可以对应到<AI>的id
self.user_tokens = self.tokenizer.encode(user_tokens)#针对不同模型，都可以对应到<用户>的id
self.assistant_tokens = self.tokenizer.encode(assistant_tokens)#针对不同模型，都可以对应到<AI>的id
self.ignore_index = -100
item = self.preprocessing(self.data[0])
print("input:", self.tokenizer.decode(item["input_ids"]))