frameworks, license, tasks, domain, tags

frameworks	license	tasks	domain	tags
Pytorch	MIT License	text-generation	nlp	fine-tuned

模型推理结果文件

见吃早饭_TouInd_12271627.jsonl

训练和推理代码

训练脚本

# Experimental environment: V100, A10, 3090
CUDA_VISIBLE_DEVICES=0 \
    swift sft \
    --model_id_or_path Qwen/Qwen2.5-7B-Instruct \
    --sft_type lora \
    --use_dora True \
    --tuner_backend peft \
    --template_type AUTO \
    --dtype AUTO \
    --output_dir output \
    --dataset qa_dataset_final3.jsonl sft_dataset_train_processed_rewritev2_with_example.jsonl \
    --num_train_epochs 3 \
    --max_length 2600 \
    --check_dataset_strategy warning \
    --lora_rank 8 \
    --lora_alpha 32 \
    --lora_dropout_p 0.05 \
    --lora_target_modules ALL \
    --gradient_checkpointing true \
    --batch_size 1 \
    --weight_decay 0.1 \
    --learning_rate 5e-5 \
    --gradient_accumulation_steps 16 \
    --max_grad_norm 0.5 \
    --warmup_ratio 0.03 \
    --eval_steps 50 \
    --save_steps 50 \
    --save_total_limit 2 \
    --logging_steps 10 

推理代码

import os

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
import gc
import hashlib
import json

import jsonlines
import torch
from swift.llm import (
    ModelType,
    get_default_template_type,
    get_model_tokenizer,
    get_template,
    get_vllm_engine,
    inference,
    inference_vllm,
)
from swift.tuners import Swift
from swift.utils import seed_everything
from tqdm import tqdm


def generate_unique_code(input_string):
    sha256_hash = hashlib.sha256()
    sha256_hash.update(input_string.encode('utf-8'))
    unique_code = sha256_hash.hexdigest()
    return unique_code

def convert_config_to_dict(config_dict):
    """Convert generation config to JSON serializable format"""
    serializable_dict = {}
    for key, value in config_dict.items():
        if isinstance(value, set):
            serializable_dict[key] = list(value)
        elif key == 'compile_config':
            # Convert CompileConfig object to dict
            serializable_dict[key] = {
                'fullgraph': value.fullgraph,
                'dynamic': value.dynamic,
                'backend': value.backend,
                'mode': value.mode,
                'options': value.options
            }
        else:
            serializable_dict[key] = value
    return serializable_dict

seed_everything(42)

file_jsonl_path = "eval_only_query.jsonl"
out_path = "吃早饭_TouInd_12271627.jsonl"
log_file = "run_log.jsonl" # 推理日志


# for objective questions
model_type = ModelType.qwen2_5_7b_instruct
template_type = get_default_template_type(model_type)
model_id_or_path = None
llm_engine = get_vllm_engine(model_type,max_model_len=10000)
llm_engine.generation_config.max_new_tokens = 1024
llm_engine.generation_config.temperature = 0
llm_engine.generation_config.do_sample = False

template = get_template(template_type, llm_engine.hf_tokenizer)

reason_data = []
with open(file_jsonl_path, "rb") as file:
    for line in jsonlines.Reader(file):
        if line["query_type"]=="objective":
            reason_data.append(
                {
                    "query": line["query"] + "\n\nAnswer this multiple-choice question step by step. First, analyze each option carefully, explain why each is correct or incorrect, and then conclude with the best answer.\n",
                    "origin": line
                }
            )
resp_list = inference_vllm(llm_engine, template, reason_data, use_tqdm=True,max_batch_size=8,)

# Add run logging for first inference
for query_data, resp in zip(reason_data, resp_list):
    run_info = {
        "unique_id": generate_unique_code(query_data["query"]),
        "model_inp": query_data["query"],
        "gen_parm": convert_config_to_dict(llm_engine.generation_config.__dict__),
        "answer": resp["response"]
    }
    with open(log_file, "a") as fw:
        fw.write(json.dumps(run_info, ensure_ascii=False)+"\n")

final_data = []
for line in resp_list:
    final_data.append(
        {
            "query": f"You are a good summarizer. Please summarize Answer and output the choise(s) like `A` or `AB` or `ACD` only.\n\n \n\nAnswer:{line['response']}\n\n",
        }
    )
# 修改max_new_tokens
resp_list = inference_vllm(llm_engine, template, final_data, use_tqdm=True,max_batch_size=8,)

# Add run logging for summarization inference
for data, resp in zip(final_data, resp_list):
    run_info = {
        "unique_id": generate_unique_code(data["query"]),
        "model_inp": data["query"],
        "gen_parm": convert_config_to_dict(llm_engine.generation_config.__dict__),
        "answer": resp["response"]
    }
    with open(log_file, "a") as fw:
        fw.write(json.dumps(run_info, ensure_ascii=False)+"\n")

with jsonlines.open(out_path, mode="a") as file_jsonl:
    for origin,resp in zip(reason_data,resp_list):
        file_jsonl.write({"query": origin["origin"]["query"], "query_type":origin["origin"]["query_type"],"answer": resp["response"]})
del llm_engine.model_executor
del llm_engine
gc.collect()
torch.cuda.empty_cache()
torch.distributed.destroy_process_group()
import ray

ray.shutdown()

# for subjective questions
ckpt_dir = "BAAI_Industry_Competition_tourism_dev/eatbreakfast_TouInd"
model_type = ModelType.qwen2_5_7b_instruct
template_type = get_default_template_type(model_type)
model_id_or_path = None
model, tokenizer = get_model_tokenizer(model_type, model_id_or_path=ckpt_dir,
                                       model_kwargs={'device_map': 'auto',"load_in_4bit":True})
# model = Swift.from_pretrained(model, ckpt_dir, inference_mode=True)

# 修改max_new_tokens
model.generation_config.max_new_tokens = 1024
model.generation_config.temperature = 0
model.generation_config.do_sample = False
template = get_template(template_type, tokenizer)

with open(file_jsonl_path,"rb") as file:
    for i,line in tqdm(enumerate(jsonlines.Reader(file))):
        if line["query_type"]=="subjective":
            query = line['query']
            response, history = inference(model, template, query)
            
            # Add run logging for subjective questions
            run_info = {
                "unique_id": generate_unique_code(query),
                "model_inp": query,
                "gen_parm": convert_config_to_dict(model.generation_config.__dict__),
                "answer": response
            }
            with open(log_file, "a") as fw:
                fw.write(json.dumps(run_info, ensure_ascii=False)+"\n")
            
            data = {"query":line["query"],"query_type":line["query_type"],"answer":response}
            with jsonlines.open(out_path,mode="a") as file_jsonl:
                file_jsonl.write(data)