11
csrc/scripts/package/common/py/__init__.py
Normal file
11
csrc/scripts/package/common/py/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
515
csrc/scripts/package/common/py/filelist.py
Normal file
515
csrc/scripts/package/common/py/filelist.py
Normal file
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
"""filelist相关类。"""
|
||||
|
||||
import itertools
|
||||
import os
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterator
|
||||
from enum import IntEnum
|
||||
from functools import partial
|
||||
from itertools import chain, repeat
|
||||
from operator import and_, attrgetter, contains, itemgetter, lt, methodcaller, ne, not_
|
||||
from typing import NamedTuple
|
||||
|
||||
from .utils.comm_log import CommLog
|
||||
from .utils.funcbase import any_, constant, dispatch, identity, invoke, pipe, side_effect, star_apply
|
||||
from .utils.pkg_utils import (
|
||||
TOP_DIR,
|
||||
FilelistError,
|
||||
GenerateFilelistError,
|
||||
conditional_apply,
|
||||
config_feature_to_string,
|
||||
pairwise,
|
||||
swap_args,
|
||||
)
|
||||
|
||||
|
||||
class FileItem(NamedTuple):
|
||||
"""文件条目"""
|
||||
|
||||
module: str
|
||||
operation: str
|
||||
relative_path_in_pkg: str
|
||||
relative_install_path: str
|
||||
is_in_docker: str
|
||||
permission: str
|
||||
owner_group: str
|
||||
install_type: str
|
||||
softlink: list[str]
|
||||
feature: set[str]
|
||||
is_common_path: str
|
||||
configurable: str
|
||||
hash_value: str
|
||||
block: str
|
||||
pkg_inner_softlink: list[str]
|
||||
chip: set[str]
|
||||
is_dir: bool
|
||||
|
||||
|
||||
def create_file_item(*args, **kwargs) -> FileItem:
|
||||
"""创建文件条目。"""
|
||||
file_item = FileItem(*args, **kwargs)
|
||||
|
||||
if not isinstance(file_item.feature, set):
|
||||
raise TypeError("The feature parameter should be a set.")
|
||||
if not isinstance(file_item.chip, set):
|
||||
raise TypeError("The chip parameter should be a set.")
|
||||
if not isinstance(file_item.softlink, list):
|
||||
raise TypeError("The softlink parameter should be a list.")
|
||||
if not isinstance(file_item.pkg_inner_softlink, list):
|
||||
raise TypeError("The pkg_inner_softlink parameter should be a list.")
|
||||
|
||||
return file_item
|
||||
|
||||
|
||||
# 文件列表
|
||||
FileList = list[FileItem]
|
||||
|
||||
|
||||
def soft_links_to_string(soft_links: list[str]) -> str:
|
||||
"""软链接转换为字符串。"""
|
||||
if not soft_links:
|
||||
return "NA"
|
||||
return ";".join(soft_links)
|
||||
|
||||
|
||||
def file_item_to_string(item: FileItem) -> str:
|
||||
"""文件条目转换为字符串。"""
|
||||
return ",".join(
|
||||
[
|
||||
item.module,
|
||||
item.operation,
|
||||
item.relative_path_in_pkg,
|
||||
item.relative_install_path,
|
||||
item.is_in_docker,
|
||||
item.permission,
|
||||
item.owner_group,
|
||||
item.install_type,
|
||||
soft_links_to_string(item.softlink),
|
||||
config_feature_to_string(item.feature),
|
||||
item.is_common_path,
|
||||
item.configurable,
|
||||
item.hash_value,
|
||||
item.block,
|
||||
soft_links_to_string(item.pkg_inner_softlink),
|
||||
config_feature_to_string(item.chip),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def get_filelist_header_string() -> str:
|
||||
"""获取文件列表表头。"""
|
||||
return ",".join(
|
||||
[
|
||||
"module",
|
||||
"operation",
|
||||
"relative_path_in_pkg",
|
||||
"relative_install_path",
|
||||
"is_in_docker",
|
||||
"permission",
|
||||
"owner:group",
|
||||
"install_type",
|
||||
"softlink",
|
||||
"feature",
|
||||
"is_common_path",
|
||||
"configurable",
|
||||
"hash",
|
||||
"block",
|
||||
"pkg_inner_softlink",
|
||||
"chip",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def get_soft_links_not_in_common_paths(filelist: FileList, target_env: str) -> Iterator[list[str]]:
|
||||
for file_item_t in filelist:
|
||||
if file_item_t.relative_install_path.startswith(target_env):
|
||||
for softlink in file_item_t.softlink:
|
||||
if not softlink.startswith(target_env):
|
||||
yield softlink
|
||||
|
||||
|
||||
def fill_is_common_path(filelist: FileList, target_env: str) -> Iterator[FileItem]:
|
||||
"""填充文件条目中是否为公共目录字段。"""
|
||||
soft_links = set(get_soft_links_not_in_common_paths(filelist, target_env))
|
||||
for file_item in filelist:
|
||||
if file_item.relative_install_path.startswith(target_env):
|
||||
yield file_item._replace(is_common_path="Y")
|
||||
else:
|
||||
is_soft_links_prefix = map(methodcaller("startswith", f"{file_item.relative_install_path}/"), soft_links)
|
||||
if any(is_soft_links_prefix):
|
||||
yield file_item._replace(is_common_path="YY")
|
||||
else:
|
||||
yield file_item
|
||||
|
||||
|
||||
def is_relative_install_path(path: str) -> bool:
|
||||
"""是否为相对路径。"""
|
||||
return not path.startswith("/")
|
||||
|
||||
|
||||
def is_specific_operations(file_item: FileItem, operations: list[str]) -> bool:
|
||||
"""是否为特定的操作类型。"""
|
||||
return file_item.operation in operations
|
||||
|
||||
|
||||
def is_specific_install_type(file_item: FileItem, install_types: set[str]) -> bool:
|
||||
"""是否为特定的安装类型。"""
|
||||
item_install_types = set(file_item.install_type.split(";"))
|
||||
if "all" in item_install_types:
|
||||
return True
|
||||
return bool(item_install_types & install_types)
|
||||
|
||||
|
||||
def get_install_path_dirs(install_path: str) -> Iterator[str]:
|
||||
"""获取安装路径父目录。"""
|
||||
install_path = os.path.dirname(install_path)
|
||||
while install_path not in ("", "/"):
|
||||
yield install_path
|
||||
install_path = os.path.dirname(install_path)
|
||||
|
||||
|
||||
def get_missing_dir_set(filelist: FileList) -> set[str]:
|
||||
"""获取缺失目录集合。
|
||||
|
||||
文件列表可能出现某一级目录缺失情况。
|
||||
如配置了file_info:aaa/bbb/ccc.txt，但只配置了dir_info:aaa，
|
||||
那么缺失dir_info:aaa/bbb
|
||||
"""
|
||||
parent_dirs: set[str] = invoke(
|
||||
pipe(
|
||||
dispatch(
|
||||
pipe(
|
||||
partial(
|
||||
filter,
|
||||
partial(is_specific_operations, operations={"copy", "copy_entity"}),
|
||||
),
|
||||
partial(map, attrgetter("relative_install_path")),
|
||||
partial(filter, is_relative_install_path),
|
||||
set,
|
||||
partial(map, get_install_path_dirs),
|
||||
chain.from_iterable,
|
||||
),
|
||||
pipe(
|
||||
partial(map, attrgetter("softlink")),
|
||||
chain.from_iterable,
|
||||
partial(
|
||||
filter,
|
||||
pipe(
|
||||
dispatch(
|
||||
bool,
|
||||
is_relative_install_path,
|
||||
partial(ne, "NA"),
|
||||
),
|
||||
all,
|
||||
),
|
||||
),
|
||||
set,
|
||||
partial(map, get_install_path_dirs),
|
||||
chain.from_iterable,
|
||||
),
|
||||
pipe(
|
||||
partial(map, attrgetter("pkg_inner_softlink")),
|
||||
chain.from_iterable,
|
||||
partial(
|
||||
filter,
|
||||
pipe(
|
||||
dispatch(
|
||||
bool,
|
||||
partial(ne, "NA"),
|
||||
),
|
||||
all,
|
||||
),
|
||||
),
|
||||
set,
|
||||
partial(map, get_install_path_dirs),
|
||||
chain.from_iterable,
|
||||
),
|
||||
),
|
||||
chain.from_iterable,
|
||||
set,
|
||||
),
|
||||
filelist,
|
||||
)
|
||||
mkdir_installs: set[str] = {
|
||||
file_item.relative_install_path
|
||||
for file_item in filter(partial(is_specific_operations, operations={"mkdir"}), filelist)
|
||||
if is_relative_install_path(file_item.relative_install_path)
|
||||
}
|
||||
|
||||
mkdir_parent_dirs: set[str] = set(itertools.chain.from_iterable(map(get_install_path_dirs, mkdir_installs)))
|
||||
|
||||
missing_dir_set = sorted((parent_dirs | mkdir_parent_dirs) - mkdir_installs)
|
||||
return set(missing_dir_set)
|
||||
|
||||
|
||||
def print_missing_dir_set(missing_dir_set: set[str], in_msg: str = None) -> set[str]:
|
||||
"""打印缺失目录集合。"""
|
||||
if in_msg:
|
||||
tail_msg = f" {in_msg}"
|
||||
else:
|
||||
tail_msg = ""
|
||||
for path in sorted(missing_dir_set):
|
||||
CommLog.cilog_error(f'missing dir info path "{path}"{tail_msg}')
|
||||
return missing_dir_set
|
||||
|
||||
|
||||
def print_unsafe_paths(unsafe_paths: tuple[str, ...]) -> tuple[str, ...]:
|
||||
"""打印非安全路径。"""
|
||||
for path in unsafe_paths:
|
||||
CommLog.cilog_error(f'unsafe path "{path}" in move scene.')
|
||||
return unsafe_paths
|
||||
|
||||
|
||||
# 获取filelist中所有的特性集合
|
||||
get_features_in_filelist = pipe(
|
||||
partial(map, attrgetter("feature")),
|
||||
chain.from_iterable, # 展开集合序列为元素序列
|
||||
set, # 去重
|
||||
partial(filter, partial(ne, "comm")), # 排除comm特性
|
||||
set,
|
||||
)
|
||||
|
||||
# 获取filelist中所有的芯片集合
|
||||
get_chips_in_filelist = pipe(
|
||||
partial(map, attrgetter("chip")),
|
||||
chain.from_iterable, # 展开集合序列为元素序列
|
||||
set, # 去重
|
||||
)
|
||||
|
||||
|
||||
def check_features_in_filelist(features: set[str], filelist: FileList) -> set[str]:
|
||||
"""检查文件列表中特性配置目录规范。"""
|
||||
return invoke(
|
||||
pipe(
|
||||
# 过滤指定features的file_item
|
||||
partial(filter, pipe(attrgetter("feature"), partial(and_, features), bool)),
|
||||
list,
|
||||
get_missing_dir_set,
|
||||
partial(print_missing_dir_set, in_msg=f"in features {features}"),
|
||||
),
|
||||
filelist,
|
||||
)
|
||||
|
||||
|
||||
def check_chip_in_filelist(chip: str, filelist: FileList) -> set[str]:
|
||||
"""检查文件列表中芯片配置目录规范。"""
|
||||
return invoke(
|
||||
pipe(
|
||||
# 过滤指定chip的file_item
|
||||
partial(
|
||||
filter,
|
||||
any_(
|
||||
pipe(attrgetter("chip"), not_), # 没有配置chip
|
||||
pipe(attrgetter("chip"), partial(swap_args(contains), chip), bool), # 配置了指定chip
|
||||
),
|
||||
),
|
||||
list,
|
||||
get_missing_dir_set,
|
||||
partial(print_missing_dir_set, in_msg=f"in chip {chip}"),
|
||||
),
|
||||
filelist,
|
||||
)
|
||||
|
||||
|
||||
check_filelist_features = any_(
|
||||
pipe(
|
||||
dispatch(
|
||||
pipe(
|
||||
get_features_in_filelist,
|
||||
# 对于每个feature，与comm组成一个set
|
||||
partial(map, lambda x: {x, "comm"}),
|
||||
# 此时为feature集合序列
|
||||
),
|
||||
repeat, # 重复filelist
|
||||
),
|
||||
tuple,
|
||||
star_apply(zip),
|
||||
# 此时为元组序列，元组的第1个元素是过滤的feature集合，第2个元素是filelist
|
||||
partial(itertools.starmap, check_features_in_filelist),
|
||||
# 此时为集合序列；合并为一个集合
|
||||
chain.from_iterable,
|
||||
set,
|
||||
),
|
||||
pipe(
|
||||
dispatch(
|
||||
get_chips_in_filelist,
|
||||
repeat, # 重复filelist
|
||||
),
|
||||
tuple,
|
||||
star_apply(zip),
|
||||
# 此时为元组序列，元组的第1个元素是chip集合，第2个元素是filelist
|
||||
partial(itertools.starmap, check_chip_in_filelist),
|
||||
# 此时为集合序列；合并为一个集合
|
||||
chain.from_iterable,
|
||||
set,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# 检查move是否安全，是否存在同一个源路径被mv多次
|
||||
check_move_safe = pipe(
|
||||
partial(
|
||||
filter,
|
||||
partial(is_specific_operations, operations={"copy", "copy_entity", "move"}),
|
||||
),
|
||||
partial(map, attrgetter("relative_path_in_pkg")),
|
||||
Counter,
|
||||
methodcaller("items"),
|
||||
partial(filter, pipe(itemgetter(1), partial(lt, 1))),
|
||||
partial(map, itemgetter(0)),
|
||||
tuple,
|
||||
print_unsafe_paths,
|
||||
)
|
||||
|
||||
|
||||
def check_filelist(filelist: FileList, check_features: bool, check_move: bool):
|
||||
"""检查文件列表是否符合规范。"""
|
||||
if check_features:
|
||||
check_features_func = check_filelist_features
|
||||
else:
|
||||
check_features_func = constant(set())
|
||||
|
||||
if check_move:
|
||||
check_move_func = check_move_safe
|
||||
else:
|
||||
check_move_func = constant(tuple())
|
||||
|
||||
# 此处使用any_，短路部分报错
|
||||
check_func = any_(
|
||||
pipe(
|
||||
get_missing_dir_set,
|
||||
print_missing_dir_set,
|
||||
),
|
||||
pipe(
|
||||
partial(filter, partial(is_specific_install_type, install_types={"run"})),
|
||||
list,
|
||||
get_missing_dir_set,
|
||||
partial(print_missing_dir_set, in_msg="in run install type"),
|
||||
),
|
||||
check_features_func,
|
||||
check_move_func,
|
||||
)
|
||||
missing = check_func(filelist)
|
||||
|
||||
if missing:
|
||||
raise FilelistError()
|
||||
|
||||
|
||||
def get_common_path(args: list[str]) -> str:
|
||||
"""公共路径前缀。"""
|
||||
try:
|
||||
return os.path.commonpath(args)
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
class FileItemRelation(IntEnum):
|
||||
"""文件条目之间的关系。"""
|
||||
|
||||
NOT_NESTED = 0 # 不是嵌套文件
|
||||
NESTED = 1 # 嵌套文件
|
||||
SAME = 2 # 相同文件
|
||||
|
||||
|
||||
def is_nested_file_item(item: FileItem, base_item: FileItem) -> FileItemRelation:
|
||||
"""是否为嵌套的文件。"""
|
||||
if base_item is None:
|
||||
return FileItemRelation.NOT_NESTED
|
||||
|
||||
if item == base_item:
|
||||
return FileItemRelation.SAME
|
||||
|
||||
install_path = item.relative_install_path
|
||||
base_install_path = base_item.relative_install_path
|
||||
|
||||
common_install_path = get_common_path([install_path, base_install_path])
|
||||
if common_install_path != base_install_path:
|
||||
return FileItemRelation.NOT_NESTED
|
||||
|
||||
pkg_path = item.relative_path_in_pkg
|
||||
base_pkg_path = base_item.relative_path_in_pkg
|
||||
|
||||
install_rel_path = os.path.relpath(install_path, base_install_path)
|
||||
pkg_rel_path = os.path.relpath(pkg_path, base_pkg_path)
|
||||
if install_rel_path != pkg_rel_path:
|
||||
# 确保打包与安装相对路径一致
|
||||
raise FilelistError(f"nested paths {item} and {base_item} are illegal.")
|
||||
return FileItemRelation.NESTED
|
||||
|
||||
|
||||
def found_nested_file_item(item: FileItem, base_item: FileItem):
|
||||
"""发现嵌套元素。"""
|
||||
raise FilelistError(f"found nested paths {item} and {base_item}!")
|
||||
|
||||
|
||||
def convert_nested_path_in_filelist(filelist: FileList):
|
||||
"""filelist中嵌套路径元素转为del。"""
|
||||
pre_item = None
|
||||
for item in filelist:
|
||||
ret = is_nested_file_item(item, pre_item)
|
||||
if ret == FileItemRelation.NESTED:
|
||||
yield item._replace(operation="del")
|
||||
elif any((ret == FileItemRelation.NOT_NESTED, (ret == FileItemRelation.SAME and not item.is_dir))):
|
||||
yield item
|
||||
pre_item = item
|
||||
|
||||
|
||||
# 检查文件列表中的嵌套路径。入参: filelist
|
||||
check_nested_path_in_filelist = pipe(
|
||||
partial(filter, partial(is_specific_operations, operations={"copy", "copy_entity"})),
|
||||
partial(sorted, key=attrgetter("relative_install_path")),
|
||||
pairwise,
|
||||
partial(map, conditional_apply(star_apply(is_nested_file_item), star_apply(found_nested_file_item))),
|
||||
list,
|
||||
)
|
||||
|
||||
|
||||
# 变换文件列表中嵌套路径。入参: filelist
|
||||
transform_nested_path_in_filelist = pipe(
|
||||
dispatch(
|
||||
partial(itertools.filterfalse, partial(is_specific_operations, operations={"copy"})),
|
||||
pipe(
|
||||
partial(filter, partial(is_specific_operations, operations={"copy"})),
|
||||
partial(sorted, key=attrgetter("relative_install_path")),
|
||||
convert_nested_path_in_filelist,
|
||||
),
|
||||
),
|
||||
chain.from_iterable,
|
||||
list,
|
||||
side_effect(check_nested_path_in_filelist),
|
||||
)
|
||||
|
||||
|
||||
def generate_filelist(filelist: FileList, filename: str):
|
||||
"""生成文件列表文件。"""
|
||||
content_list = list(
|
||||
itertools.chain([get_filelist_header_string()], [file_item_to_string(item) for item in filelist])
|
||||
)
|
||||
content = "\n".join(content_list)
|
||||
filepath = os.path.join(TOP_DIR, "build", filename)
|
||||
try:
|
||||
with open(filepath, "w", encoding="utf-8") as file:
|
||||
file.write(content)
|
||||
# filelist.csv文件末尾补充一个换行符
|
||||
file.write("\n")
|
||||
except OSError as ex:
|
||||
raise GenerateFilelistError(filename) from ex
|
||||
|
||||
|
||||
def get_transform_nested_path_func(parallel: bool) -> Callable[[FileList], FileList]:
|
||||
"""获取转换嵌套路径函数。"""
|
||||
if parallel:
|
||||
return transform_nested_path_in_filelist
|
||||
return identity
|
||||
69
csrc/scripts/package/common/py/merge_binary_info_config.py
Normal file
69
csrc/scripts/package/common/py/merge_binary_info_config.py
Normal file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
"""合并算子binary_info_config.json。"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def load_json_file(json_file: str):
|
||||
"""加载json文件。"""
|
||||
with open(json_file, encoding="utf-8") as file:
|
||||
json_content = json.load(file)
|
||||
return json_content
|
||||
|
||||
|
||||
def save_json_file(output_file: str, content):
|
||||
"""保存json文件。"""
|
||||
output_dir = os.path.dirname(output_file)
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
with open(output_file, "w", encoding="utf-8") as file:
|
||||
json.dump(content, file, ensure_ascii=True, indent=2)
|
||||
|
||||
|
||||
def update_config(base_content, update_content):
|
||||
"""更新配置。"""
|
||||
new_content = base_content.copy()
|
||||
new_content.update(update_content)
|
||||
return dict(sorted(new_content.items()))
|
||||
|
||||
|
||||
def parse_args(argv: list[str]):
|
||||
"""入参解析。"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-file", required=True, help="the basic binary_info_config file")
|
||||
parser.add_argument("--update-file", required=True, help="the update binary_info_config file")
|
||||
parser.add_argument(
|
||||
"--output-file", required=True, type=os.path.realpath, help="the output binary_info_config file"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
return args
|
||||
|
||||
|
||||
def main(argv: list[str]) -> bool:
|
||||
"""主流程。"""
|
||||
args = parse_args(argv)
|
||||
base_content = load_json_file(args.base_file)
|
||||
update_content = load_json_file(args.update_file)
|
||||
result = update_config(base_content, update_content)
|
||||
save_json_file(args.output_file, result)
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not main(sys.argv[1:]): # pragma: no cover
|
||||
sys.exit(1) # pragma: no cover
|
||||
240
csrc/scripts/package/common/py/packer.py
Normal file
240
csrc/scripts/package/common/py/packer.py
Normal file
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from argparse import Namespace
|
||||
from collections.abc import Callable
|
||||
from itertools import chain
|
||||
from subprocess import PIPE, STDOUT
|
||||
from typing import NamedTuple
|
||||
|
||||
from .utils.comm_log import CommLog
|
||||
from .utils.pkg_utils import CompressError
|
||||
|
||||
|
||||
class PackageName:
|
||||
"""包名。"""
|
||||
|
||||
def __init__(self, package_attr, args: Namespace, version: str):
|
||||
self.product_name = package_attr.get("product_name")
|
||||
self.chip_name = args.chip_name or package_attr.get("chip_name")
|
||||
self.suffix = args.suffix or package_attr.get("suffix")
|
||||
self.func_name = get_func_name(args.func_name, package_attr)
|
||||
self.chip_plat = package_attr.get("chip_plat")
|
||||
self.deploy_type = package_attr.get("deploy_type")
|
||||
self.version = version.lower()
|
||||
self.not_in_name_list = args.not_in_name.split(",")
|
||||
self.os_arch = args.os_arch
|
||||
self.package_suffix = args.package_suffix
|
||||
self.ext_name = args.ext_name
|
||||
if args.pkg_name_style == "underline":
|
||||
self.name_sep = "_"
|
||||
else:
|
||||
self.name_sep = "-"
|
||||
|
||||
def get_attribute(self, name: str) -> str | None:
|
||||
"""获取属性。"""
|
||||
if name in self.not_in_name_list:
|
||||
return None
|
||||
return getattr(self, name)
|
||||
|
||||
def getvalue(self) -> str:
|
||||
product_name = self.get_attribute("product_name")
|
||||
chip_name = self.get_attribute("chip_name")
|
||||
func_name = self.get_attribute("func_name")
|
||||
version = self.get_attribute("version")
|
||||
os_arch = self.get_attribute("os_arch")
|
||||
chip_plat = self.get_attribute("chip_plat")
|
||||
deploy_type = self.get_attribute("deploy_type")
|
||||
ext_name = self.get_attribute("ext_name")
|
||||
package_suffix = "debug" if self.package_suffix == "debug" else None
|
||||
|
||||
region1 = "-".join(filter(None, [product_name, remove_ascend(chip_name), func_name]))
|
||||
region2 = ".".join(filter(None, [version]))
|
||||
region3 = "-".join(filter(None, [os_arch, chip_plat, deploy_type, package_suffix, ext_name]))
|
||||
package_name = "_".join(filter(None, [region1, region2, region3]))
|
||||
|
||||
return f"{package_name}.{self.suffix}"
|
||||
|
||||
|
||||
class MakeselfPkgParams(NamedTuple):
|
||||
"""run包打包参数。"""
|
||||
|
||||
package_name: str
|
||||
comments: str
|
||||
makeself_tool: str | None = None
|
||||
makeself_header: str | None = None
|
||||
help_info: str | None = None
|
||||
source_target: str | None = None
|
||||
|
||||
install_script: str | None = None
|
||||
independent_pkg: bool | None = False
|
||||
cleanup: str | None = None
|
||||
|
||||
|
||||
def remove_ascend(text):
|
||||
if text is None:
|
||||
return None
|
||||
text_lower = text.lower()
|
||||
if text_lower == "ascend910_93":
|
||||
return "A3"
|
||||
if "ascend" in text_lower:
|
||||
return text_lower.replace("ascend", "")
|
||||
return text_lower
|
||||
|
||||
|
||||
def get_func_name(func_name: str, package_attr) -> str:
|
||||
"""获取包func_name。"""
|
||||
return func_name or package_attr.get("func_name")
|
||||
|
||||
|
||||
def get_compress_tool() -> str:
|
||||
tools = ["pigz", "gzip", "bzip2", "xz"]
|
||||
for tool in tools:
|
||||
path = shutil.which(tool)
|
||||
if path:
|
||||
return "--" + tool
|
||||
CommLog.cilog_error(
|
||||
"The system does not come with a compression tool pre-installed."
|
||||
"Please ensure at least one of the following compression tools is available: %s",
|
||||
tools,
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def get_compress_format() -> str:
|
||||
tar_format = "gnu"
|
||||
path = shutil.which("bsdtar")
|
||||
if path:
|
||||
tar_format = "ustar"
|
||||
return tar_format
|
||||
|
||||
|
||||
def compose_makeself_command(params: MakeselfPkgParams) -> str:
|
||||
"""组装makeself包打包命令。"""
|
||||
|
||||
def get_cleanup_commands() -> list[str]:
|
||||
if params.cleanup:
|
||||
return ["--cleanup", params.cleanup]
|
||||
return []
|
||||
|
||||
independent_pkg = params.independent_pkg
|
||||
compress_tool = get_compress_tool()
|
||||
tar_format = get_compress_format()
|
||||
if independent_pkg:
|
||||
commands = chain(
|
||||
[
|
||||
"TMPDIR=$pwd",
|
||||
params.makeself_tool,
|
||||
"--header",
|
||||
params.makeself_header,
|
||||
"--help-header",
|
||||
params.help_info,
|
||||
compress_tool,
|
||||
"--complevel",
|
||||
"4",
|
||||
"--nomd5",
|
||||
"--sha256",
|
||||
"--nooverwrite",
|
||||
"--chown",
|
||||
"--tar-format",
|
||||
tar_format,
|
||||
"--tar-extra",
|
||||
"--numeric-owner",
|
||||
"--tar-quietly",
|
||||
],
|
||||
get_cleanup_commands(),
|
||||
[params.source_target, params.package_name, params.comments, params.install_script],
|
||||
)
|
||||
else:
|
||||
commands = chain(
|
||||
[
|
||||
compress_tool,
|
||||
"--complevel",
|
||||
"4",
|
||||
"--nomd5",
|
||||
"--sha256",
|
||||
"--nooverwrite",
|
||||
"--chown",
|
||||
"--tar-format",
|
||||
tar_format,
|
||||
"--tar-extra",
|
||||
"--numeric-owner",
|
||||
"--tar-quietly",
|
||||
],
|
||||
get_cleanup_commands(),
|
||||
[params.package_name, params.comments],
|
||||
)
|
||||
|
||||
command = " ".join(commands)
|
||||
return command
|
||||
|
||||
|
||||
def create_makeself_pkg_params_factory(
|
||||
source_target: str, package_name: str, comments: str
|
||||
) -> Callable[[str, dict, bool], MakeselfPkgParams]:
|
||||
"""创建Makeself打包参数工厂。"""
|
||||
|
||||
def create_makeself_pkg_params(makeself_dir: str, package_attr: dict, independent_pkg=False) -> MakeselfPkgParams:
|
||||
"""创建Makeself打包参数。"""
|
||||
cleanup = package_attr.get("cleanup")
|
||||
|
||||
if independent_pkg:
|
||||
install_script = str(package_attr.get("install_script"))
|
||||
help_info = str(package_attr.get("help"))
|
||||
makeself_tool = os.path.join(makeself_dir, "makeself.sh")
|
||||
makeself_header = os.path.join(makeself_dir, "makeself-header.sh")
|
||||
params = MakeselfPkgParams(
|
||||
package_name=package_name,
|
||||
comments=comments,
|
||||
makeself_tool=makeself_tool,
|
||||
makeself_header=makeself_header,
|
||||
help_info=help_info,
|
||||
source_target=source_target,
|
||||
install_script=install_script,
|
||||
independent_pkg=independent_pkg,
|
||||
cleanup=cleanup,
|
||||
)
|
||||
else:
|
||||
params = MakeselfPkgParams(
|
||||
package_name=package_name,
|
||||
comments=comments,
|
||||
cleanup=cleanup,
|
||||
)
|
||||
return params
|
||||
|
||||
return create_makeself_pkg_params
|
||||
|
||||
|
||||
def create_run_package_command(params: MakeselfPkgParams) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
功能描述: 组装打run包命令
|
||||
返回值: command
|
||||
"""
|
||||
return compose_makeself_command(params), None
|
||||
|
||||
|
||||
def exec_pack_cmd(delivery_dir: str, pack_cmd: str, package_name: str) -> str:
|
||||
"""执行打包命令"""
|
||||
if delivery_dir:
|
||||
cmd = f"cd {delivery_dir} && {pack_cmd}"
|
||||
else:
|
||||
cmd = pack_cmd
|
||||
CommLog.cilog_info("package cmd:%s", cmd)
|
||||
result = subprocess.run(cmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT)
|
||||
output = result.stdout.decode()
|
||||
if result.returncode != 0:
|
||||
CommLog.cilog_error(__file__, "compress package(%s) failed! %s.", package_name, output)
|
||||
raise CompressError(package_name)
|
||||
return package_name
|
||||
1058
csrc/scripts/package/common/py/pkg_parser.py
Normal file
1058
csrc/scripts/package/common/py/pkg_parser.py
Normal file
File diff suppressed because it is too large
Load Diff
64
csrc/scripts/package/common/py/utils/comm_log.py
Normal file
64
csrc/scripts/package/common/py/utils/comm_log.py
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import time
|
||||
|
||||
logging.basicConfig(
|
||||
format="[%(asctime)s] [%(levelname)s] [%(pathname)s] [line:%(lineno)d] %(message)s", level=logging.INFO
|
||||
)
|
||||
|
||||
|
||||
class CommLog:
|
||||
@staticmethod
|
||||
def cilog_get_timestamp():
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||||
|
||||
@staticmethod
|
||||
def cilog_print_element(cilog_element):
|
||||
print("[" + cilog_element + "]", end=" ")
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def cilog_logmsg(log_level, filename, line_no, log_msg, *log_paras):
|
||||
log_timestamp = CommLog.cilog_get_timestamp()
|
||||
CommLog.cilog_print_element(log_timestamp)
|
||||
CommLog.cilog_print_element(log_level)
|
||||
CommLog.cilog_print_element(filename)
|
||||
CommLog.cilog_print_element(str(line_no))
|
||||
print(log_msg % log_paras[0])
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def cilog_error(log_msg, *log_paras):
|
||||
frame = inspect.currentframe().f_back
|
||||
line_no = frame.f_lineno
|
||||
filename = frame.f_code.co_filename
|
||||
CommLog.cilog_logmsg("ERROR", filename, line_no, log_msg, log_paras)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def cilog_warning(log_msg, *log_paras):
|
||||
frame = inspect.currentframe().f_back
|
||||
line_no = frame.f_lineno
|
||||
filename = frame.f_code.co_filename
|
||||
CommLog.cilog_logmsg("WARNING", filename, line_no, log_msg, log_paras)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def cilog_info(log_msg, *log_paras):
|
||||
frame = inspect.currentframe().f_back
|
||||
line_no = frame.f_lineno
|
||||
filename = frame.f_code.co_filename
|
||||
CommLog.cilog_logmsg("INFO", filename, line_no, log_msg, log_paras)
|
||||
return
|
||||
94
csrc/scripts/package/common/py/utils/funcbase.py
Normal file
94
csrc/scripts/package/common/py/utils/funcbase.py
Normal file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
"""函数基础库。"""
|
||||
|
||||
import operator
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import TypeVar
|
||||
|
||||
A = TypeVar("A")
|
||||
|
||||
|
||||
def constant(value: A) -> Callable[..., A]:
|
||||
"""常量值。"""
|
||||
|
||||
def constant_inner(*_args, **_kwargs) -> A:
|
||||
return value
|
||||
|
||||
return constant_inner
|
||||
|
||||
|
||||
def dispatch(*funcs):
|
||||
"""分派应用。"""
|
||||
|
||||
def dispatch_inner(*args, **kwargs) -> Iterator:
|
||||
return (func(*args, **kwargs) for func in funcs)
|
||||
|
||||
return dispatch_inner
|
||||
|
||||
|
||||
def pipe(*funcs):
|
||||
"""串联多个函数。"""
|
||||
|
||||
def pipe_func(*args, **k_args):
|
||||
result = funcs[0](*args, **k_args)
|
||||
for func in funcs[1:]:
|
||||
result = func(result)
|
||||
return result
|
||||
|
||||
return pipe_func
|
||||
|
||||
|
||||
def identity(value: A) -> A:
|
||||
"""同一。"""
|
||||
return value
|
||||
|
||||
|
||||
def invoke(func, *args, **kwargs):
|
||||
"""调用。"""
|
||||
return func(*args, **kwargs)
|
||||
|
||||
|
||||
def side_effect(*funcs):
|
||||
"""调用函数，产生副作用，但不影响管道结果。"""
|
||||
|
||||
def side_effect_func(arg):
|
||||
for func in funcs:
|
||||
# 不保留结果
|
||||
func(arg)
|
||||
return arg
|
||||
|
||||
return side_effect_func
|
||||
|
||||
|
||||
def star_apply(func):
|
||||
"""列表展开再应用。"""
|
||||
|
||||
def star_apply_func(arg):
|
||||
return func(*arg)
|
||||
|
||||
return star_apply_func
|
||||
|
||||
|
||||
def any_(*funcs) -> Callable:
|
||||
"""高阶any。
|
||||
注意，any有短路效果。"""
|
||||
return pipe(
|
||||
dispatch(*funcs),
|
||||
any,
|
||||
)
|
||||
|
||||
|
||||
def not_(func) -> Callable:
|
||||
"""高阶not。"""
|
||||
return pipe(func, operator.not_)
|
||||
181
csrc/scripts/package/common/py/utils/pkg_utils.py
Normal file
181
csrc/scripts/package/common/py/utils/pkg_utils.py
Normal file
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
"""基础构件。"""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from itertools import chain, tee
|
||||
from pathlib import Path
|
||||
from typing import Optional, TypeVar
|
||||
|
||||
TOP_DIR = str(Path(__file__).resolve().parents[5])
|
||||
TOP_SOURCE_DIR = TOP_DIR + "/scripts/"
|
||||
DELIVERY_PATH = "build/_CPack_Packages/makeself_staging"
|
||||
CONFIG_SCRIPT_PATH = "package"
|
||||
BLOCK_CONFIG_PATH = "package/module"
|
||||
|
||||
SUCCESS = 0
|
||||
FAIL = -1
|
||||
|
||||
|
||||
A = TypeVar("A")
|
||||
|
||||
|
||||
class PackageError(Exception):
|
||||
"""打包异常基类。"""
|
||||
|
||||
|
||||
class PackageConfigError(PackageError):
|
||||
"""打包配置错误异常。"""
|
||||
|
||||
|
||||
class BlockConfigError(PackageError):
|
||||
"""块配置错误异常。"""
|
||||
|
||||
|
||||
class ParseOsArchError(PackageError):
|
||||
"""解析os_arch失败异常。"""
|
||||
|
||||
|
||||
class EnvNotSupported(PackageError):
|
||||
"""环境变量不支持异常。"""
|
||||
|
||||
|
||||
class ContainAsteriskError(PackageError):
|
||||
"""包含星号异常。"""
|
||||
|
||||
def __init__(self, value: str):
|
||||
super().__init__()
|
||||
self.value = value
|
||||
|
||||
|
||||
class FilelistError(PackageError):
|
||||
"""文件列表异常。"""
|
||||
|
||||
|
||||
class UnknownOperateTypeError(PackageError):
|
||||
"""未知的操作类型。"""
|
||||
|
||||
|
||||
class PackageNameEmptyError(PackageError):
|
||||
"""包名为空错误。"""
|
||||
|
||||
|
||||
class GenerateFilelistError(PackageError):
|
||||
"""生成文件列表文件异常。"""
|
||||
|
||||
def __init__(self, filename: str):
|
||||
super().__init__()
|
||||
self.filename = filename
|
||||
|
||||
|
||||
class IllegalVersionDir(PackageError):
|
||||
"""version_dir配置错误。"""
|
||||
|
||||
|
||||
class CompressError(PackageError):
|
||||
"""打包错误。"""
|
||||
|
||||
def __init__(self, package_name: str | None):
|
||||
super().__init__(package_name)
|
||||
self.package_name = package_name
|
||||
|
||||
|
||||
def flatten(list_of_lists):
|
||||
"""Flatten one level of nesting"""
|
||||
return chain.from_iterable(list_of_lists)
|
||||
|
||||
|
||||
def merge_dict(base: dict, *news: dict):
|
||||
"""合并两个字典。"""
|
||||
result = base.copy()
|
||||
for new in news:
|
||||
result.update(new)
|
||||
return result
|
||||
|
||||
|
||||
def star_pipe(*funcs):
|
||||
"""串联多个函数。解包结果。"""
|
||||
|
||||
def pipe_func(*args, **k_args):
|
||||
result = funcs[0](*args, **k_args)
|
||||
for func in funcs[1:]:
|
||||
# 解包元组或列表结果
|
||||
result = func(*result)
|
||||
return result
|
||||
|
||||
return pipe_func
|
||||
|
||||
|
||||
def swap_args(func):
|
||||
"""交换函数前两个参数。"""
|
||||
|
||||
def inner(fst, snd, *args, **k_args):
|
||||
return func(snd, fst, *args, **k_args)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
def conditional_apply(predicate, func):
|
||||
"""条件下应用函数。"""
|
||||
|
||||
def conditional_apply_func(arg):
|
||||
if predicate(arg):
|
||||
return func(arg)
|
||||
return arg
|
||||
|
||||
return conditional_apply_func
|
||||
|
||||
|
||||
def pairwise(iterable):
|
||||
"""s -> (s0,s1), (s1,s2), (s2, s3), ..."""
|
||||
a, b = tee(iterable)
|
||||
next(b, None)
|
||||
return zip(a, b)
|
||||
|
||||
|
||||
def path_join(base: Optional, *others: str) -> Optional:
|
||||
"""路径联合。"""
|
||||
if base is None:
|
||||
return None
|
||||
return os.path.join(base, *others)
|
||||
|
||||
|
||||
def yield_if(data, predicate: Callable) -> Iterator:
|
||||
"""条件满足则产生。"""
|
||||
if predicate(data):
|
||||
yield data
|
||||
|
||||
|
||||
def config_feature_to_set(feature_str: str, feature_type: str = "feature") -> set[str]:
|
||||
"""配置feature转换为集合。"""
|
||||
if feature_str is None:
|
||||
return set()
|
||||
|
||||
if isinstance(feature_str, set):
|
||||
return feature_str
|
||||
|
||||
if feature_str == "":
|
||||
raise PackageConfigError(f"Not allow to config {feature_type} empty.")
|
||||
|
||||
features = set(feature_str.split(";"))
|
||||
if "all" in features:
|
||||
raise PackageConfigError(f"Not allow to config {feature_type} all.")
|
||||
return features
|
||||
|
||||
|
||||
def config_feature_to_string(features: set[str]) -> str:
|
||||
"""配置feature集合转换为字符串。"""
|
||||
if not features:
|
||||
return "all"
|
||||
return ";".join(sorted(features))
|
||||
435
csrc/scripts/package/common/py/version_info.py
Normal file
435
csrc/scripts/package/common/py/version_info.py
Normal file
@@ -0,0 +1,435 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
|
||||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
from functools import total_ordering
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import regex as re
|
||||
|
||||
|
||||
class VersionInfoError(Exception):
|
||||
"""版本信息异常基类。"""
|
||||
|
||||
|
||||
class VersionFormatNotMatch(VersionInfoError):
|
||||
"""版本格式未匹配。"""
|
||||
|
||||
|
||||
class IntervalFormatNotMatch(VersionInfoError):
|
||||
"""区间格式未匹配。"""
|
||||
|
||||
|
||||
class DuplicatedPkgConfig(VersionInfoError):
|
||||
"""解析版本配置失败。重复的包配置。"""
|
||||
|
||||
def __init__(self, pkg_name):
|
||||
super().__init__(pkg_name)
|
||||
self.pkg_name = pkg_name
|
||||
|
||||
|
||||
class ParseVersionFailed(VersionInfoError):
|
||||
"""解析版本失败。"""
|
||||
|
||||
|
||||
class CollectRequiresFailed(VersionInfoError):
|
||||
"""收集包需求失败。"""
|
||||
|
||||
def __init__(self, pkg_name, version_str, msg):
|
||||
super().__init__(pkg_name, version_str, msg)
|
||||
self.pkg_name = pkg_name
|
||||
self.version_str = version_str
|
||||
self.msg = msg
|
||||
|
||||
|
||||
@total_ordering
|
||||
class Version:
|
||||
"""版本号。"""
|
||||
|
||||
def __init__(self, version):
|
||||
self.version = version
|
||||
|
||||
@classmethod
|
||||
def match(cls, input_str):
|
||||
"""输入字符串是否匹配版本号模式。"""
|
||||
m = re.match(r"[.a-zA-Z0-9]+$", input_str)
|
||||
return bool(m)
|
||||
|
||||
@classmethod
|
||||
def parse(cls, input_str):
|
||||
"""解析版本号。"""
|
||||
if not cls.match(input_str):
|
||||
raise VersionFormatNotMatch()
|
||||
|
||||
return cls(input_str)
|
||||
|
||||
@classmethod
|
||||
def try_convert_to_int_list(cls, str_list):
|
||||
"""尝试转换为int数组。"""
|
||||
for idx, item in enumerate(str_list):
|
||||
try:
|
||||
int_item = int(item)
|
||||
str_list[idx] = int_item
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def to_required_list(self):
|
||||
"""转换为版本需求字符串列表。"""
|
||||
return [self.version]
|
||||
|
||||
def __eq__(self, other):
|
||||
"""等于。"""
|
||||
if not isinstance(other, self.__class__):
|
||||
return False
|
||||
return self.version == other.version
|
||||
|
||||
def __lt__(self, other):
|
||||
"""小于。"""
|
||||
if not isinstance(other, self.__class__):
|
||||
return True
|
||||
|
||||
self_list = self.version.split(".")
|
||||
other_list = other.version.split(".")
|
||||
|
||||
self.try_convert_to_int_list(self_list)
|
||||
self.try_convert_to_int_list(other_list)
|
||||
|
||||
self_tuple = tuple(self_list)
|
||||
other_tuple = tuple(other_list)
|
||||
|
||||
return self_tuple < other_tuple
|
||||
|
||||
def __str__(self):
|
||||
return self.version
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.version)
|
||||
|
||||
|
||||
class Point(NamedTuple):
|
||||
"""区间端点。"""
|
||||
|
||||
type_: int # 类型，0为闭区间，1为开区间
|
||||
value: Version
|
||||
|
||||
|
||||
class Interval(NamedTuple):
|
||||
"""版本号区间。"""
|
||||
|
||||
low: Point
|
||||
high: Point
|
||||
|
||||
@classmethod
|
||||
def match(cls, input_str: str) -> bool:
|
||||
"""输入字符串是否匹配区间模式。"""
|
||||
if not input_str.startswith("(") and not input_str.startswith("["):
|
||||
return False
|
||||
if not input_str.endswith(")") and not input_str.endswith("]"):
|
||||
return False
|
||||
input_str = input_str[1:-1]
|
||||
return input_str.count(",") <= 1
|
||||
|
||||
@classmethod
|
||||
def parse(cls, input_str):
|
||||
"""解析版本号区间。"""
|
||||
if not cls.match(input_str):
|
||||
raise IntervalFormatNotMatch()
|
||||
|
||||
if input_str[0] == "[":
|
||||
low_type = 0
|
||||
elif input_str[0] == "(":
|
||||
low_type = 1
|
||||
else:
|
||||
raise AssertionError("should not go here.")
|
||||
|
||||
if input_str[-1] == "]":
|
||||
high_type = 0
|
||||
elif input_str[-1] == ")":
|
||||
high_type = 1
|
||||
else:
|
||||
raise AssertionError("should not go here.")
|
||||
|
||||
input_str = input_str[1:-1]
|
||||
input_list = input_str.split(",")
|
||||
low = input_list[0].strip()
|
||||
if len(input_list) > 1:
|
||||
high = input_list[1].strip()
|
||||
else:
|
||||
high = None
|
||||
|
||||
if low:
|
||||
low_version = Point(low_type, Version(low))
|
||||
else:
|
||||
low_version = None
|
||||
|
||||
if high:
|
||||
high_version = Point(high_type, Version(high))
|
||||
else:
|
||||
high_version = None
|
||||
|
||||
return cls(low=low_version, high=high_version)
|
||||
|
||||
def to_required_list(self):
|
||||
"""转换为版本需求字符串列表。"""
|
||||
result = []
|
||||
|
||||
if self.low:
|
||||
if self.low.type_ == 0:
|
||||
operator = ">="
|
||||
else:
|
||||
operator = ">"
|
||||
required_str = f"{operator}{self.low.value.version}"
|
||||
result.append(required_str)
|
||||
|
||||
if self.high:
|
||||
if self.high.type_ == 0:
|
||||
operator = "<="
|
||||
else:
|
||||
operator = "<"
|
||||
required_str = f"{operator}{self.high.value.version}"
|
||||
result.append(required_str)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class Require(NamedTuple):
|
||||
"""包需求。"""
|
||||
|
||||
pkg_name: str
|
||||
versions: list
|
||||
|
||||
@classmethod
|
||||
def _sort_key(cls, item) -> tuple:
|
||||
"""排序键。"""
|
||||
if isinstance(item, Interval):
|
||||
# 如果存在区间左值，则左值参与排序。
|
||||
if item.low:
|
||||
return item.low.value, item.low.type_
|
||||
# 否则使用区间右值，由于开区间更小，所以type_取负。
|
||||
return item.high.value, -item.high.type_
|
||||
|
||||
return item, 0
|
||||
|
||||
@classmethod
|
||||
def _sort_versions(cls, versions: list) -> bool:
|
||||
"""排序版本序列。"""
|
||||
versions.sort(key=cls._sort_key)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _to_required_list(cls, versions: list) -> list[str]:
|
||||
"""转换为版本需求字符串列表。"""
|
||||
result = []
|
||||
for version in versions:
|
||||
requires = version.to_required_list()
|
||||
result.extend(requires)
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _to_required_str(cls, versions: list) -> str:
|
||||
"""转换为版本需求字符串。"""
|
||||
requires = cls._to_required_list(versions)
|
||||
required_str = ", ".join(requires)
|
||||
|
||||
return required_str
|
||||
|
||||
def sort_versions(self) -> bool:
|
||||
"""排序版本序列。"""
|
||||
return self._sort_versions(self.versions)
|
||||
|
||||
def to_required_full_str(self) -> str:
|
||||
"""转换为版本需求字符串。"""
|
||||
required_str = self._to_required_str(self.versions)
|
||||
required_full_str = f'required_package_{self.pkg_name}_version="{required_str}"'
|
||||
return required_full_str
|
||||
|
||||
|
||||
class ItemElement(NamedTuple):
|
||||
"""item元素。"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
|
||||
@classmethod
|
||||
def parse(cls, item_ele: ET.Element, cur_ver: str):
|
||||
"""解析item元素。"""
|
||||
name = item_ele.attrib["name"]
|
||||
version = item_ele.attrib["version"].replace("$(CUR_VER)", cur_ver)
|
||||
return cls(name=name, version=version)
|
||||
|
||||
@classmethod
|
||||
def skip(cls, item_ele: ET.Element):
|
||||
"""是否跳过item元素。"""
|
||||
version = item_ele.attrib["version"]
|
||||
return version.strip() == ""
|
||||
|
||||
|
||||
class CompatibleElement(NamedTuple):
|
||||
"""compatible元素。"""
|
||||
|
||||
items: list
|
||||
|
||||
@classmethod
|
||||
def parse(cls, compatible_ele: ET.Element, cur_ver: str):
|
||||
"""解析compatible元素。"""
|
||||
items = []
|
||||
for item_ele in compatible_ele.findall("./item"):
|
||||
if ItemElement.skip(item_ele):
|
||||
continue
|
||||
item = ItemElement.parse(item_ele, cur_ver)
|
||||
items.append(item)
|
||||
return cls(items=items)
|
||||
|
||||
|
||||
def is_version_number(version: str) -> bool:
|
||||
"""字符串是否为版本号。"""
|
||||
has_slash = "/" in version
|
||||
return not has_slash and len(version.split(".")) >= 3
|
||||
|
||||
|
||||
class VersionXml(NamedTuple):
|
||||
"""版本配置。"""
|
||||
|
||||
release_version: str
|
||||
version_dir: str
|
||||
packages: dict
|
||||
|
||||
@classmethod
|
||||
def match(cls, filepath: Path | str) -> bool:
|
||||
"""文件路径是否匹配版本信息文件。"""
|
||||
return str(filepath).endswith(".xml")
|
||||
|
||||
@classmethod
|
||||
def parse_version(cls, version_str: str):
|
||||
"""解析版本配置。"""
|
||||
ret = Interval.match(version_str)
|
||||
if ret:
|
||||
result = Interval.parse(version_str)
|
||||
return result
|
||||
|
||||
ret = Version.match(version_str)
|
||||
if ret:
|
||||
result = Version.parse(version_str)
|
||||
return result
|
||||
|
||||
raise ParseVersionFailed()
|
||||
|
||||
def get_release_version(self):
|
||||
"""获取发布版本号。"""
|
||||
return self.release_version
|
||||
|
||||
def get_version_dir(self):
|
||||
"""获取多版本目录。"""
|
||||
return self.version_dir
|
||||
|
||||
def collect_requires(self, package: str) -> list[Require]:
|
||||
"""收集对应包的包需求列表。"""
|
||||
requires = {}
|
||||
|
||||
if package not in self.packages:
|
||||
return []
|
||||
|
||||
compatible = self.packages[package]
|
||||
|
||||
for item in compatible.items:
|
||||
pkg_name = item.name
|
||||
if pkg_name not in requires:
|
||||
requires[pkg_name] = Require(pkg_name=pkg_name, versions=[])
|
||||
|
||||
version_str = item.version
|
||||
try:
|
||||
version = self.parse_version(version_str)
|
||||
except ParseVersionFailed as ex:
|
||||
msg = f"parse pkg {pkg_name} version {version_str} failed"
|
||||
raise CollectRequiresFailed(pkg_name, version_str, msg) from ex
|
||||
|
||||
requires[pkg_name].versions.append(version)
|
||||
|
||||
result = []
|
||||
for pkg_name in sorted(requires.keys()):
|
||||
requires[pkg_name].sort_versions()
|
||||
result.append(requires[pkg_name])
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_version_dir(version_xml: VersionXml | None, disable_multi_version: bool, version_dir: str | None) -> str | None:
|
||||
"""获取版本目录名。"""
|
||||
if disable_multi_version:
|
||||
return None
|
||||
|
||||
if version_dir:
|
||||
return version_dir
|
||||
|
||||
# 支持从version.xml中获取version_dir
|
||||
if version_xml and version_xml.get_version_dir():
|
||||
return version_xml.get_version_dir()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_multi_version(version_dir: str) -> bool:
|
||||
"""是否多版本。"""
|
||||
return bool(version_dir)
|
||||
|
||||
|
||||
class VersionInfo(NamedTuple):
|
||||
"""版本信息。"""
|
||||
|
||||
install_version_info: bool
|
||||
install_version_info_attrib: dict[str, str] | None
|
||||
itf_versions: list[str]
|
||||
version: str
|
||||
version_xml: VersionXml | None
|
||||
timestamp: str | None
|
||||
|
||||
|
||||
class VersionInfoFile(NamedTuple):
|
||||
"""生成的版本配置。"""
|
||||
|
||||
version: str
|
||||
itf_version_info: str | None = None
|
||||
requires: list[Require] | None = None
|
||||
version_dir: str | None = None
|
||||
timestamp: str | None = None
|
||||
|
||||
def _get_content(self) -> str:
|
||||
"""获取版本配置内容。"""
|
||||
lines = [f"Version={self.version}"]
|
||||
if self.version_dir:
|
||||
lines.append(f"version_dir={self.version_dir}")
|
||||
if self.timestamp:
|
||||
lines.append(f"timestamp={self.timestamp}")
|
||||
if self.itf_version_info:
|
||||
lines.append(self.itf_version_info)
|
||||
|
||||
if self.requires:
|
||||
requires_str = [require.to_required_full_str() for require in self.requires]
|
||||
lines.extend(requires_str)
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def save(self, target_path: Path | str):
|
||||
"""保存版本配置。"""
|
||||
content = self._get_content()
|
||||
|
||||
target_dir = os.path.dirname(target_path)
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
|
||||
with open(target_path, "w") as file:
|
||||
file.write(content)
|
||||
Reference in New Issue
Block a user