--- frameworks: - Pytorch license: MIT License tasks: - text-generation #model-type: ##如 gpt、phi、llama、chatglm、baichuan 等 #- gpt domain: - nlp #language: ##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa #- cn #metrics: ##如 CIDEr、Blue、ROUGE 等 #- CIDEr tags: ##各种自定义，包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他 - fine-tuned #tools: ##如 vllm、fastchat、llamacpp、AdaSeq 等 #- vllm --- ## 模型推理结果文件 见`吃早饭_TouInd_12271627.jsonl` ## 训练和推理代码 ### 训练脚本 ```bash # Experimental environment: V100, A10, 3090 CUDA_VISIBLE_DEVICES=0 \ swift sft \ --model_id_or_path Qwen/Qwen2.5-7B-Instruct \ --sft_type lora \ --use_dora True \ --tuner_backend peft \ --template_type AUTO \ --dtype AUTO \ --output_dir output \ --dataset qa_dataset_final3.jsonl sft_dataset_train_processed_rewritev2_with_example.jsonl \ --num_train_epochs 3 \ --max_length 2600 \ --check_dataset_strategy warning \ --lora_rank 8 \ --lora_alpha 32 \ --lora_dropout_p 0.05 \ --lora_target_modules ALL \ --gradient_checkpointing true \ --batch_size 1 \ --weight_decay 0.1 \ --learning_rate 5e-5 \ --gradient_accumulation_steps 16 \ --max_grad_norm 0.5 \ --warmup_ratio 0.03 \ --eval_steps 50 \ --save_steps 50 \ --save_total_limit 2 \ --logging_steps 10 ``` ### 推理代码 ```python import os os.environ['CUDA_VISIBLE_DEVICES'] = '0' import gc import hashlib import json import jsonlines import torch from swift.llm import ( ModelType, get_default_template_type, get_model_tokenizer, get_template, get_vllm_engine, inference, inference_vllm, ) from swift.tuners import Swift from swift.utils import seed_everything from tqdm import tqdm def generate_unique_code(input_string): sha256_hash = hashlib.sha256() sha256_hash.update(input_string.encode('utf-8')) unique_code = sha256_hash.hexdigest() return unique_code def convert_config_to_dict(config_dict): """Convert generation config to JSON serializable format""" serializable_dict = {} for key, value in config_dict.items(): if isinstance(value, set): serializable_dict[key] = list(value) elif key == 'compile_config': # Convert CompileConfig object to dict serializable_dict[key] = { 'fullgraph': value.fullgraph, 'dynamic': value.dynamic, 'backend': value.backend, 'mode': value.mode, 'options': value.options } else: serializable_dict[key] = value return serializable_dict seed_everything(42) file_jsonl_path = "eval_only_query.jsonl" out_path = "吃早饭_TouInd_12271627.jsonl" log_file = "run_log.jsonl" # 推理日志 # for objective questions model_type = ModelType.qwen2_5_7b_instruct template_type = get_default_template_type(model_type) model_id_or_path = None llm_engine = get_vllm_engine(model_type,max_model_len=10000) llm_engine.generation_config.max_new_tokens = 1024 llm_engine.generation_config.temperature = 0 llm_engine.generation_config.do_sample = False template = get_template(template_type, llm_engine.hf_tokenizer) reason_data = [] with open(file_jsonl_path, "rb") as file: for line in jsonlines.Reader(file): if line["query_type"]=="objective": reason_data.append( { "query": line["query"] + "\n\nAnswer this multiple-choice question step by step. First, analyze each option carefully, explain why each is correct or incorrect, and then conclude with the best answer.\n", "origin": line } ) resp_list = inference_vllm(llm_engine, template, reason_data, use_tqdm=True,max_batch_size=8,) # Add run logging for first inference for query_data, resp in zip(reason_data, resp_list): run_info = { "unique_id": generate_unique_code(query_data["query"]), "model_inp": query_data["query"], "gen_parm": convert_config_to_dict(llm_engine.generation_config.__dict__), "answer": resp["response"] } with open(log_file, "a") as fw: fw.write(json.dumps(run_info, ensure_ascii=False)+"\n") final_data = [] for line in resp_list: final_data.append( { "query": f"You are a good summarizer. Please summarize Answer and output the choise(s) like `A` or `AB` or `ACD` only.\n\n \n\nAnswer:{line['response']}\n\n", } ) # 修改max_new_tokens resp_list = inference_vllm(llm_engine, template, final_data, use_tqdm=True,max_batch_size=8,) # Add run logging for summarization inference for data, resp in zip(final_data, resp_list): run_info = { "unique_id": generate_unique_code(data["query"]), "model_inp": data["query"], "gen_parm": convert_config_to_dict(llm_engine.generation_config.__dict__), "answer": resp["response"] } with open(log_file, "a") as fw: fw.write(json.dumps(run_info, ensure_ascii=False)+"\n") with jsonlines.open(out_path, mode="a") as file_jsonl: for origin,resp in zip(reason_data,resp_list): file_jsonl.write({"query": origin["origin"]["query"], "query_type":origin["origin"]["query_type"],"answer": resp["response"]}) del llm_engine.model_executor del llm_engine gc.collect() torch.cuda.empty_cache() torch.distributed.destroy_process_group() import ray ray.shutdown() # for subjective questions ckpt_dir = "BAAI_Industry_Competition_tourism_dev/eatbreakfast_TouInd" model_type = ModelType.qwen2_5_7b_instruct template_type = get_default_template_type(model_type) model_id_or_path = None model, tokenizer = get_model_tokenizer(model_type, model_id_or_path=ckpt_dir, model_kwargs={'device_map': 'auto',"load_in_4bit":True}) # model = Swift.from_pretrained(model, ckpt_dir, inference_mode=True) # 修改max_new_tokens model.generation_config.max_new_tokens = 1024 model.generation_config.temperature = 0 model.generation_config.do_sample = False template = get_template(template_type, tokenizer) with open(file_jsonl_path,"rb") as file: for i,line in tqdm(enumerate(jsonlines.Reader(file))): if line["query_type"]=="subjective": query = line['query'] response, history = inference(model, template, query) # Add run logging for subjective questions run_info = { "unique_id": generate_unique_code(query), "model_inp": query, "gen_parm": convert_config_to_dict(model.generation_config.__dict__), "answer": response } with open(log_file, "a") as fw: fw.write(json.dumps(run_info, ensure_ascii=False)+"\n") data = {"query":line["query"],"query_type":line["query_type"],"answer":response} with jsonlines.open(out_path,mode="a") as file_jsonl: file_jsonl.write(data) ```