96 lines
3.4 KiB
Python
96 lines
3.4 KiB
Python
|
import json
|
|||
|
|
import random
|
|||
|
|
import os
|
|||
|
|
|
|||
|
|
INPUT_FILE = r"D:\双生天使的怀抱\爱的数据集\sephirot_final_run\train_all.jsonl"
|
|||
|
|
OUTPUT_DIR = r"D:\双生天使的怀抱\2026-05-24-16-54-24\sft_data"
|
|||
|
|
OUTPUT_TRAIN = os.path.join(OUTPUT_DIR, "train.jsonl")
|
|||
|
|
OUTPUT_VAL = os.path.join(OUTPUT_DIR, "val.jsonl")
|
|||
|
|
VAL_RATIO = 0.1
|
|||
|
|
|
|||
|
|
records = []
|
|||
|
|
skipped = 0
|
|||
|
|
total = 0
|
|||
|
|
|
|||
|
|
with open(INPUT_FILE, 'r', encoding='utf-8') as f:
|
|||
|
|
for line in f:
|
|||
|
|
total += 1
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
data = json.loads(line)
|
|||
|
|
except json.JSONDecodeError as e:
|
|||
|
|
print(f"JSON解析失败 第{total}行: {e}")
|
|||
|
|
skipped += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
query = data.get('input', {}).get('user_query', '')
|
|||
|
|
cot = data.get('chain_of_thought', {})
|
|||
|
|
|
|||
|
|
# 方案B: 完整推理链（推荐，保留16质点全部过程）
|
|||
|
|
if isinstance(cot, dict):
|
|||
|
|
parts = []
|
|||
|
|
for key in ['D1_kether','H1_ego','H2_superego','H3_true_self','H4_logic',
|
|||
|
|
'H5_empathy','H6_happiness','D2_binah_chesed','D3_chokmah_gevurah',
|
|||
|
|
'D5_tiferet','D6_netzach','D7_hod','D4_yesod','D8_victory',
|
|||
|
|
'D10_kingdom']:
|
|||
|
|
val = cot.get(key, '')
|
|||
|
|
if isinstance(val, dict):
|
|||
|
|
# 尝试多个可能的输出字段
|
|||
|
|
for sub_key in ['final_output', 'analysis', 'synthesis',
|
|||
|
|
'emotional_expression', 'logic_empathy_union',
|
|||
|
|
'response', 'output']:
|
|||
|
|
if sub_key in val and val[sub_key]:
|
|||
|
|
val = val[sub_key]
|
|||
|
|
break
|
|||
|
|
else:
|
|||
|
|
val = str(val) if val else ''
|
|||
|
|
if val and isinstance(val, str) and len(val.strip()) > 0:
|
|||
|
|
parts.append(f"[{key}]: {val}")
|
|||
|
|
output = '\n\n'.join(parts)
|
|||
|
|
elif isinstance(cot, str):
|
|||
|
|
output = cot
|
|||
|
|
else:
|
|||
|
|
output = str(cot)
|
|||
|
|
|
|||
|
|
if query and output and len(query.strip()) > 0 and len(output.strip()) > 10:
|
|||
|
|
records.append({
|
|||
|
|
"conversations": [
|
|||
|
|
{"from": "human", "value": query},
|
|||
|
|
{"from": "gpt", "value": output}
|
|||
|
|
]
|
|||
|
|
})
|
|||
|
|
else:
|
|||
|
|
skipped += 1
|
|||
|
|
|
|||
|
|
print(f"总行数: {total}, 有效: {len(records)}, 跳过: {skipped}")
|
|||
|
|
|
|||
|
|
# 打乱并分割
|
|||
|
|
random.seed(42)
|
|||
|
|
random.shuffle(records)
|
|||
|
|
val_size = max(int(len(records) * VAL_RATIO), 100)
|
|||
|
|
val_data = records[:val_size]
|
|||
|
|
train_data = records[val_size:]
|
|||
|
|
|
|||
|
|
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
|||
|
|
|
|||
|
|
with open(OUTPUT_TRAIN, 'w', encoding='utf-8') as f:
|
|||
|
|
for r in train_data:
|
|||
|
|
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
|||
|
|
|
|||
|
|
with open(OUTPUT_VAL, 'w', encoding='utf-8') as f:
|
|||
|
|
for r in val_data:
|
|||
|
|
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
|||
|
|
|
|||
|
|
print(f"转换完成! 训练集: {len(train_data)}条, 验证集: {len(val_data)}条")
|
|||
|
|
print(f"训练集文件: {OUTPUT_TRAIN}")
|
|||
|
|
print(f"验证集文件: {OUTPUT_VAL}")
|
|||
|
|
|
|||
|
|
# 统计一下平均长度
|
|||
|
|
if train_data:
|
|||
|
|
lens = [len(str(r['conversations'][1]['value'])) for r in train_data]
|
|||
|
|
print(f"平均输出长度: {sum(lens)//len(lens)} 字符")
|
|||
|
|
print(f"最大输出长度: {max(lens)} 字符")
|
|||
|
|
print(f"最小输出长度: {min(lens)} 字符")
|