RYDE-WORK/MiniCPM
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/MiniCPM.git synced 2026-01-19 12:53:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

注释前加了空格，保持代码规范性，刘丹

Browse Source
This commit is contained in:
root 2024-06-21 16:19:01 +08:00
parent 062ea5264a
commit b808010417
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
finetune/finetune.py
View File

@ -55,8 +55,8 @@ class SupervisedDataset(Dataset):
self.data = json.load(open(data_path))
self.tokenizer = tokenizer
self.model_max_length = model_max_length
self.user_tokens = self.tokenizer.encode(user_tokens)#针对不同模型，都可以对应到<用户>的id
self.assistant_tokens = self.tokenizer.encode(assistant_tokens)#针对不同模型，都可以对应到<AI>的id
self.user_tokens = self.tokenizer.encode(user_tokens) #针对不同模型，都可以对应到<用户>的id
self.assistant_tokens = self.tokenizer.encode(assistant_tokens) #针对不同模型，都可以对应到<AI>的id
self.ignore_index = -100
item = self.preprocessing(self.data[0])
print("input:", self.tokenizer.decode(item["input_ids"]))