import gc import json import os import sys import time import zipfile import yaml from schemas.context import ASRContext from utils.client import Client from utils.evaluator import BaseEvaluator from utils.logger import logger from utils.service import register_sut IN_TEST = os.getenv("SUBMIT_CONFIG_FILEPATH", None) is None UNIT_TEST = os.getenv("UNIT_TEST", 0) def main(): logger.info("执行……") dataset_filepath = os.getenv( "DATASET_FILEPATH", "./tests/resources/en.zip", ) submit_config_filepath = os.getenv("SUBMIT_CONFIG_FILEPATH", "./tests/resources/submit_config") result_filepath = os.getenv("RESULT_FILEPATH", "./out/result") bad_cases_filepath = os.getenv("BAD_CASES_FILEPATH", "./out/badcase") detail_cases_filepath = os.getenv("DETAILED_CASES_FILEPATH", "./out/detailcase.jsonl") resource_name = os.getenv("BENCHMARK_NAME") # 提交配置 & 启动被测服务 if os.getenv("DATASET_FILEPATH", ""): from utils.helm import resource_check with open(submit_config_filepath, "r") as fp: st_config = yaml.safe_load(fp) st_config["values"] = resource_check(st_config.get("values", {})) if 'docker_images' in st_config: sut_url = "ws://172.26.1.75:9827" os.environ['test'] = '1' elif 'docker_image' in st_config: sut_url = register_sut(st_config, resource_name) elif UNIT_TEST: sut_url = "ws://172.27.231.36:80" else: logger.error("config 配置错误,没有 docker_image") os._exit(1) else: os.environ['test'] = '1' sut_url = "ws://172.27.231.36:80" if UNIT_TEST: exit(0) """ # 数据集处理 local_dataset_path = "./dataset" os.makedirs(local_dataset_path, exist_ok=True) with zipfile.ZipFile(dataset_filepath) as zf: zf.extractall(local_dataset_path) config_path = os.path.join(local_dataset_path, "data.yaml") with open(config_path, "r") as fp: dataset_config = yaml.safe_load(fp) # 数据集信息 dataset_global_config = dataset_config.get("global", {}) dataset_query = dataset_config.get("query_data", {}) evaluator = BaseEvaluator() # 开始预测 for idx, query_item in enumerate(dataset_query): gc.collect() logger.info(f"开始执行 {idx} 条数据") context = ASRContext(**dataset_global_config) context.lang = query_item.get("lang", context.lang) context.file_path = os.path.join(local_dataset_path, query_item["file"]) # context.audio_length = query_item["audio_length"] interactions = Client(sut_url, context).action() context.append_labels(query_item["voice"]) context.append_preds( interactions["predict_data"], interactions["send_time"], interactions["recv_time"], ) context.fail = interactions["fail"] if IN_TEST: with open('output.txt', 'w') as fp: original_stdout = sys.stdout sys.stdout = fp print(context) sys.stdout = original_stdout evaluator.evaluate(context) detail_case = evaluator.gen_detail_case() with open(detail_cases_filepath, "a") as fp: fp.write(json.dumps(detail_case.to_dict(), ensure_ascii=False) + "\n") time.sleep(4) evaluator.post_evaluate() output_result = evaluator.gen_result() # print(evaluator.__dict__) logger.info("执行完成. Result = {output_result}") with open(result_filepath, "w") as fp: json.dump(output_result, fp, indent=2, ensure_ascii=False) with open(bad_cases_filepath, "w") as fp: fp.write("当前榜单不存在 Bad Case\n") """ if __name__ == "__main__": main()