1059 lines
36 KiB
Python
1059 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: UTF-8 -*-
|
||
# -----------------------------------------------------------------------------------------------------------
|
||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||
# See LICENSE in the root of the software repository for the full text of the License.
|
||
# -----------------------------------------------------------------------------------------------------------
|
||
|
||
import copy
|
||
import glob
|
||
import hashlib
|
||
import itertools
|
||
import os
|
||
import sys
|
||
import xml.etree.ElementTree as ET
|
||
from argparse import Namespace
|
||
from collections.abc import Callable, Iterable, Iterator
|
||
from functools import partial
|
||
from io import StringIO
|
||
from itertools import chain
|
||
from operator import attrgetter, itemgetter, methodcaller
|
||
from typing import Any, NamedTuple
|
||
|
||
import regex as re
|
||
|
||
from .filelist import FileItem, FileList, fill_is_common_path
|
||
from .utils import pkg_utils
|
||
from .utils.comm_log import CommLog
|
||
from .utils.funcbase import constant, dispatch, invoke, pipe, star_apply
|
||
from .utils.pkg_utils import (
|
||
BLOCK_CONFIG_PATH,
|
||
FAIL,
|
||
BlockConfigError,
|
||
ContainAsteriskError,
|
||
EnvNotSupported,
|
||
IllegalVersionDir,
|
||
PackageError,
|
||
ParseOsArchError,
|
||
config_feature_to_set,
|
||
flatten,
|
||
merge_dict,
|
||
star_pipe,
|
||
yield_if,
|
||
)
|
||
from .version_info import VersionFormatNotMatch, VersionInfo, VersionXml, is_multi_version
|
||
|
||
# 环境变量字典
|
||
EnvDict = dict[str, str]
|
||
|
||
# 文件信息
|
||
FileInfo = dict[str, str]
|
||
|
||
# 包属性
|
||
PackageAttr = dict[str, str | bool]
|
||
|
||
# 生成信息
|
||
GenerateInfo = dict[str, str]
|
||
|
||
|
||
class ParseOption(NamedTuple):
|
||
"""解析参数。"""
|
||
|
||
os_arch: str | None
|
||
pkg_version: str | None
|
||
build_type: str | None
|
||
package_check: bool
|
||
ext_name: str = ""
|
||
|
||
|
||
def parse_os_arch(os_arch: str) -> tuple[str, str, str]:
|
||
"""解析系统和架构。"""
|
||
match = re.match("^([a-z]+)(\\d+(\\.\\d+)*)?[.-]?(\\S*)", os_arch)
|
||
if match:
|
||
os_name = match.group(1)
|
||
os_ver = match.group(2)
|
||
if match.group(4):
|
||
arch = match.group(4)
|
||
else:
|
||
# 如果os_arch中没有配置ARCH，ARCH默认值为aarch64
|
||
arch = "aarch64"
|
||
|
||
return os_name, os_ver, arch
|
||
|
||
raise ParseOsArchError()
|
||
|
||
|
||
def replace_env(env_dict: EnvDict, in_str: str):
|
||
"""替换环境变量为实际值。"""
|
||
env_list = re.findall(".*?\\$\\((.*?)\\).*?", in_str)
|
||
for env in env_list:
|
||
if env == "FILE":
|
||
continue
|
||
if env in env_dict:
|
||
if env_dict[env] is not None:
|
||
in_str = in_str.replace(f"$({env})", env_dict[env])
|
||
else:
|
||
in_str = in_str.replace(f"$({env})", "")
|
||
else:
|
||
raise EnvNotSupported(f"Error: {env} not supported.")
|
||
return in_str
|
||
|
||
|
||
class ParseEnv(NamedTuple):
|
||
"""解析上下文环境。"""
|
||
|
||
env_dict: EnvDict
|
||
parse_option: ParseOption
|
||
delivery_dir: str
|
||
top_dir: str
|
||
|
||
|
||
class BlockElement(NamedTuple):
|
||
"""块配置。"""
|
||
|
||
name: str
|
||
block_conf_path: str
|
||
dst_path: str
|
||
chips: set[str]
|
||
features: set[str]
|
||
attrs: dict[str, str]
|
||
|
||
|
||
# BlockElement直接透传给LoadedBlockElement的参数列表
|
||
BLOCK_ELEMENT_PASS_THROUGH_ARGS = ["dst_path", "chips", "features", "attrs"]
|
||
|
||
|
||
class LoadedBlockElement(NamedTuple):
|
||
"""加载后的块配置。"""
|
||
|
||
root_ele: ET.Element
|
||
use_move: bool
|
||
dst_path: str
|
||
chips: set[str]
|
||
features: set[str]
|
||
attrs: dict[str, str]
|
||
|
||
|
||
class FileInfoParsedResult(NamedTuple):
|
||
"""file_info元素解析结果。"""
|
||
|
||
file_info: FileInfo
|
||
move_infos: list[FileInfo]
|
||
dir_infos: list[dict[str, str]]
|
||
expand_infos: list[dict[str, str]]
|
||
|
||
|
||
class BlockConfig(NamedTuple):
|
||
"""块配置。"""
|
||
|
||
dir_install_list: list[dict]
|
||
move_files: list[FileInfo]
|
||
expand_content_list: list[dict]
|
||
package_content_list: list[dict]
|
||
generate_infos: list[GenerateInfo]
|
||
|
||
|
||
class PackerConfig(NamedTuple):
|
||
"""安装相关配置。"""
|
||
|
||
fill_is_common_path: Callable[[FileList], Iterator[FileItem]]
|
||
|
||
|
||
class XmlConfig(NamedTuple):
|
||
"""安装xml配置。"""
|
||
|
||
default_config: dict[str, str]
|
||
package_attr: PackageAttr
|
||
version_info: VersionInfo
|
||
blocks: list[BlockConfig]
|
||
version: str
|
||
version_xml: VersionXml | None
|
||
packer_config: PackerConfig
|
||
|
||
def _collect_list(self, list_name):
|
||
result = []
|
||
for block in self.blocks:
|
||
result.extend(getattr(block, list_name))
|
||
return result
|
||
|
||
@property
|
||
def dir_install_list(self):
|
||
return self._collect_list("dir_install_list")
|
||
|
||
@property
|
||
def move_content_list(self):
|
||
return self._collect_list("move_files")
|
||
|
||
@property
|
||
def expand_content_list(self):
|
||
return self._collect_list("expand_content_list")
|
||
|
||
@property
|
||
def package_content_list(self):
|
||
return self._collect_list("package_content_list")
|
||
|
||
@property
|
||
def generate_infos(self) -> list[GenerateInfo]:
|
||
return self._collect_list("generate_infos")
|
||
|
||
|
||
# 默认包属性
|
||
DEFAULT_PACKAGE_ATTR = {
|
||
"gen_version_info": True,
|
||
}
|
||
|
||
|
||
def parse_package_info(package_info_ele: ET.Element | None) -> dict:
|
||
"""解析package_info元素。"""
|
||
|
||
def get_package_info_attrs(ele: ET.Element) -> Iterator[tuple[str, str | bool]]:
|
||
# expand_asterisk: 展开配置中星号
|
||
# parallel: 并行复制文件
|
||
# parallel_limit: 限制并发数
|
||
# package_check: 检查filelist.csv中配置目录是否完整
|
||
# check_features: 检查filelist.csv中所有feature是否符合package_check
|
||
# gen_version_info: 是否生成version.info文件
|
||
bool_attrs = (
|
||
"expand_asterisk",
|
||
"parallel",
|
||
"parallel_limit",
|
||
"package_check",
|
||
"check_features",
|
||
"use_move",
|
||
"gen_version_info",
|
||
)
|
||
bool_values = ("t", "true", "y", "yes")
|
||
if ele.tag in bool_attrs:
|
||
if ele.text.lower() in bool_values:
|
||
yield ele.tag, True
|
||
else:
|
||
yield ele.tag, False
|
||
else:
|
||
yield ele.tag, ele.text
|
||
|
||
if not package_info_ele:
|
||
return {}
|
||
|
||
attr = dict(chain.from_iterable(map(get_package_info_attrs, list(package_info_ele))))
|
||
|
||
return attr
|
||
|
||
|
||
def parse_package_attr_by_args(args: Namespace) -> dict:
|
||
"""通过命令行参数解析"""
|
||
|
||
def pairs():
|
||
if hasattr(args, "chip_name") and args.chip_name:
|
||
yield "chip_name", args.chip_name
|
||
if hasattr(args, "suffix") and args.suffix:
|
||
yield "suffix", args.suffix
|
||
if hasattr(args, "func_name") and args.func_name:
|
||
yield "func_name", args.func_name
|
||
|
||
return dict(pairs())
|
||
|
||
|
||
def parse_package_attr(root_ele: ET.Element, args: Namespace) -> dict:
|
||
"""通过根元素解析package_info元素。"""
|
||
package_info_ele = root_ele.find("package_info")
|
||
return merge_dict(
|
||
DEFAULT_PACKAGE_ATTR,
|
||
parse_package_info(package_info_ele),
|
||
parse_package_attr_by_args(args),
|
||
)
|
||
|
||
|
||
def render_cann_version(
|
||
a_ver: int, b_ver: int, c_ver: int | None, d_ver: int | None, e_ver: int | None, f_ver: int | None
|
||
) -> str:
|
||
"""渲染CANN版本号。"""
|
||
buffer = StringIO()
|
||
buffer.write("(")
|
||
buffer.write(f"({a_ver + 1} * 100000000) + ({b_ver + 1} * 1000000)")
|
||
if c_ver is not None:
|
||
buffer.write(f" + ({c_ver + 1} * 10000)")
|
||
if d_ver is not None:
|
||
buffer.write(f" + (({d_ver + 1} * 100) + 5000)")
|
||
if e_ver is not None:
|
||
buffer.write(f" + ({e_ver + 1} * 100)")
|
||
if f_ver is not None:
|
||
buffer.write(f" + {f_ver}")
|
||
buffer.write(")")
|
||
return buffer.getvalue()
|
||
|
||
|
||
def render_semver(package_name: str, version: str) -> Iterator[tuple[str, str]]:
|
||
"""
|
||
将语义化版本号转换为可比较的整数表达式，严格遵循SemVer规范
|
||
|
||
排序规则：
|
||
1. 正式版本 > 所有对应预发布版本（如 8.0.5 > 8.0.5-rc.1）
|
||
2. 预发布类型优先级：rc > beta > alpha > 其他类型（如 rc.1 > beta.100）
|
||
3. 同类型预发布版本：序号越大优先级越高（如 alpha.10 > alpha.2）
|
||
4. 多段序号比较：从左到右逐段比较（如 alpha.1.2 > alpha.1.1）
|
||
"""
|
||
expr_buffer = StringIO()
|
||
expr_buffer.write("(")
|
||
|
||
# 移除构建元数据（+后面的内容不影响版本优先级）
|
||
version = version.split("+")[0]
|
||
|
||
# 分离正式版本和预发布版本
|
||
pre_release = None
|
||
if "-" in version:
|
||
release_part, pre_release = version.split("-", 1)
|
||
release_part = release_part.split(".")
|
||
else:
|
||
release_part = version.split(".")
|
||
if len(release_part) > 3:
|
||
pre_release = ".".join(release_part[3:])
|
||
release_part = release_part[:3]
|
||
|
||
# 解析正式版本号（主版本.次版本.修订号）
|
||
try:
|
||
major, minor, patch = map(int, release_part)
|
||
except (ValueError, TypeError) as ex:
|
||
raise IllegalVersionDir(f"无效的版本号格式: {version}") from ex
|
||
|
||
yield f"{package_name}_VERSION_STR", f'"{version}"'
|
||
yield f"{package_name}_MAJOR", str(major)
|
||
yield f"{package_name}_MINOR", str(minor)
|
||
yield f"{package_name}_PATCH", str(patch)
|
||
# 计算基础版本值（主版本*10^7 + 次版本*10^5 + 修订号*10^3）
|
||
# 预留10^3空间用于预发布版本，确保不同正式版本区间不重叠
|
||
expr_buffer.write(f"({major} * 10000000) + ({minor} * 100000) + ({patch} * 1000)")
|
||
|
||
# 处理正式版本（无预发布部分）
|
||
if not pre_release:
|
||
expr_buffer.write(")")
|
||
yield f"{package_name}_PRERELEASE", '""'
|
||
yield f"{package_name}_VERSION_NUM", expr_buffer.getvalue()
|
||
return
|
||
|
||
yield f"{package_name}_PRERELEASE", f'"{pre_release}"'
|
||
|
||
# 预发布类型权重（值越小优先级越高）
|
||
type_weights = {
|
||
"rc": 100, # rc优先级最高
|
||
"beta": 200, # beta次之
|
||
"alpha": 300, # alpha最低
|
||
}
|
||
|
||
def calc_pre_release() -> tuple[int, int]:
|
||
"""计算预发布版本。"""
|
||
if "." in pre_release:
|
||
pre_parts = pre_release.split(".")
|
||
pre_type = pre_parts[0] # 提取预发布类型（rc/beta/alpha等）
|
||
|
||
# 提取序号部分（支持多段序号，非数字部分忽略）
|
||
pre_nums = []
|
||
for part in pre_parts[1:]:
|
||
if part.isdigit():
|
||
pre_nums.append(int(part))
|
||
if not pre_nums: # 无序号时默认0
|
||
pre_nums = [0]
|
||
|
||
# 未知类型权重设为400（优先级低于alpha）
|
||
pre_type_weight = type_weights.get(pre_type, 400)
|
||
|
||
# 计算序号值（支持多段和多位数）
|
||
num_str = "".join(map(str, pre_nums))
|
||
# 转换为整数并返回
|
||
num_value = int(num_str)
|
||
return pre_type_weight, num_value
|
||
|
||
for pre_type in type_weights:
|
||
if pre_release.startswith(pre_type):
|
||
pre_type_weight = type_weights[pre_type]
|
||
num_value = int(pre_release[len(pre_type) :])
|
||
return pre_type_weight, num_value
|
||
|
||
return None, None
|
||
|
||
try:
|
||
pre_type_weight, num_value = calc_pre_release()
|
||
except (ValueError, TypeError) as ex:
|
||
raise IllegalVersionDir(f"无效的预发布版本: {pre_release}") from ex
|
||
|
||
if not pre_type_weight:
|
||
raise IllegalVersionDir(f"无效的预发布版本: {pre_release}")
|
||
|
||
# 预发布版本最终值 = 基础值 - 类型权重 + 序号值
|
||
# 确保：预发布值 < 基础值（正式版本）
|
||
expr_buffer.write(f" - {pre_type_weight} + {num_value}")
|
||
expr_buffer.write(")")
|
||
|
||
yield f"{package_name}_VERSION_NUM", expr_buffer.getvalue()
|
||
|
||
|
||
def get_cann_version_info(name: str, version: str) -> Iterator[tuple[str, str]]:
|
||
"""获取CANN版本号信息。"""
|
||
# 删除字符串中的_VERSION
|
||
package_name = name[:-8]
|
||
|
||
if not version:
|
||
yield f"{package_name}_VERSION_STR", '"0"'
|
||
return
|
||
|
||
yield from render_semver(package_name, version)
|
||
|
||
|
||
def get_default_env_items() -> Iterator[tuple[str, str]]:
|
||
"""获取默认环境字典条目。"""
|
||
yield "VERSION_DIR", ""
|
||
yield "HOME", os.environ.get("HOME")
|
||
|
||
|
||
def get_env_items_by_version(version: str | None) -> Iterator[tuple[str, str]]:
|
||
"""根据version获取环境字典条目。"""
|
||
if version:
|
||
yield "ASCEND_VER", version
|
||
|
||
version_parts = version.split(".")
|
||
for idx in range(1, len(version_parts) + 1):
|
||
yield f"CUR_VER[{idx}]", ".".join(version_parts[:idx])
|
||
yield "CUR_VER", version
|
||
yield "LOWER_CUR_VER", version.lower()
|
||
|
||
|
||
def get_env_items_by_version_dir(version_dir: str | None) -> Iterator[tuple[str, str]]:
|
||
"""根据version_dir获取环境字典条目。"""
|
||
if version_dir:
|
||
yield "VERSION_DIR", version_dir
|
||
|
||
|
||
def get_os_arch_default_env_items() -> Iterator[tuple[str, str]]:
|
||
"""获取系统相关默认环境字典条目。"""
|
||
yield "OS_NAME", "linux"
|
||
yield "OS_VER", ""
|
||
yield "ARM", "aarch64"
|
||
yield "TARGET_ENV", "$(TARGET_ENV)"
|
||
|
||
|
||
def get_env_items_by_os_arch(os_arch: str) -> Iterator[tuple[str, str]]:
|
||
"""根据os_arch获取环境字典条目。"""
|
||
if os_arch:
|
||
os_name, os_ver, arch = parse_os_arch(os_arch)
|
||
yield "OS_NAME", os_name
|
||
yield "OS_VER", os_ver
|
||
yield "ARCH", arch
|
||
yield "OS_ARCH", os_arch
|
||
if arch in ("arm", "sw_64"):
|
||
yield "ARM", arch
|
||
else:
|
||
yield "ARM", "aarch64"
|
||
yield "TARGET_ENV", f"{arch}-linux"
|
||
else:
|
||
yield from get_os_arch_default_env_items()
|
||
|
||
|
||
def get_env_items_by_timestamp(timestamp: str | None) -> Iterator[tuple[str, str]]:
|
||
"""根据timestamp获取环境字典条目。"""
|
||
if timestamp:
|
||
yield "TIMESTAMP", timestamp
|
||
yield "TIMESTAMP_NO", timestamp.replace("_", "")
|
||
else:
|
||
yield "TIMESTAMP", "0"
|
||
yield "TIMESTAMP_NO", "0"
|
||
|
||
|
||
def parse_env_dict(
|
||
os_arch: str, package_attr: PackageAttr, version: str | None, version_dir: str | None, timestamp: str | None
|
||
) -> EnvDict:
|
||
"""解析环境变量字典。"""
|
||
env_dict = dict(
|
||
chain(
|
||
get_default_env_items(),
|
||
yield_if(("ARCH", package_attr.get("default_arch")), itemgetter(1)),
|
||
get_env_items_by_os_arch(os_arch),
|
||
get_env_items_by_version(version),
|
||
get_env_items_by_version_dir(version_dir),
|
||
yield_if(("VERSION_DIR", version_dir), constant(version_dir)),
|
||
get_env_items_by_timestamp(timestamp),
|
||
)
|
||
)
|
||
|
||
return env_dict
|
||
|
||
|
||
def get_timestamp(args: Namespace) -> str | None:
|
||
"""获取触发时间戳。"""
|
||
if "tag" not in args:
|
||
return None
|
||
|
||
tag = args.tag
|
||
if tag:
|
||
timestamp_re = r"\d{8}_\d{9}"
|
||
timestamp_list = re.findall(timestamp_re, tag)
|
||
if not timestamp_list:
|
||
raise PackageError("The {} format is incorrect.".format(tag))
|
||
timestamp = timestamp_list[-1]
|
||
else:
|
||
timestamp = None
|
||
return timestamp
|
||
|
||
|
||
def extract_element_attrib(ele: ET.Element) -> dict:
|
||
"""提取元素属性。"""
|
||
return ele.attrib.copy()
|
||
|
||
|
||
def extract_generate_info_content(generate_info_ele: ET.Element, env_dict: EnvDict) -> dict:
|
||
"""提取生成信息内容。"""
|
||
file_content = {sub_item.tag: replace_env(env_dict, sub_item.text) for sub_item in list(generate_info_ele)}
|
||
return {"content": file_content}
|
||
|
||
|
||
def parse_generate_infos_by_loaded_block(
|
||
loaded_block: LoadedBlockElement, default_config: dict[str, str], env_dict: EnvDict
|
||
) -> list[dict]:
|
||
"""根据根元素解析生成信息列表。"""
|
||
return invoke(
|
||
pipe(
|
||
partial(
|
||
map,
|
||
pipe(
|
||
dispatch(
|
||
pipe(
|
||
extract_element_attrib,
|
||
partial(merge_dict, default_config),
|
||
partial(evaluate_info, loaded_block=loaded_block, env_dict=env_dict),
|
||
),
|
||
partial(extract_generate_info_content, env_dict=env_dict),
|
||
),
|
||
star_apply(merge_dict),
|
||
),
|
||
),
|
||
list,
|
||
),
|
||
loaded_block.root_ele.findall("generate_info"),
|
||
)
|
||
|
||
|
||
def join_pkg_inner_softlink(link_str_list: list[str]) -> str:
|
||
"""合并pkg_inner_softlink"""
|
||
path = "/".join(link_str_list)
|
||
return os.path.normpath(path)
|
||
|
||
|
||
def check_contain_asterisk(value: str) -> bool:
|
||
"""检查串是否包含星号。"""
|
||
return "*" in value
|
||
|
||
|
||
def check_value(value: str, package_check: bool, package_attr: PackageAttr):
|
||
"""检查元素value属性。"""
|
||
if package_check and package_attr.get("suffix") == "run":
|
||
if check_contain_asterisk(value):
|
||
raise ContainAsteriskError(value)
|
||
|
||
|
||
def get_dst_prefix(file_info: FileInfo, env: ParseEnv) -> str:
|
||
"""获取文件的前缀。"""
|
||
return os.path.join(env.delivery_dir, file_info["dst_path"])
|
||
|
||
|
||
def get_dst_target(file_info: FileInfo, env: ParseEnv) -> str:
|
||
"""获取文件的实际路径。"""
|
||
dst_prefix = get_dst_prefix(file_info, env)
|
||
return os.path.join(dst_prefix, os.path.basename(file_info.get("value")))
|
||
|
||
|
||
def make_hash(filepath: str) -> str:
|
||
"""计算文件的hash(sha256)值。"""
|
||
sha256_hash = hashlib.sha256()
|
||
with open(filepath, "rb") as file:
|
||
sha256_hash.update(file.read())
|
||
|
||
return sha256_hash.hexdigest()
|
||
|
||
|
||
def config_hash(parsed_result: FileInfoParsedResult, env: ParseEnv):
|
||
"""配置hash值。"""
|
||
file_info = parsed_result.file_info
|
||
# 如果配置了configurable，需要计算文件的hash值
|
||
if file_info and file_info["configurable"] == "TRUE":
|
||
src_target = get_dst_target(file_info, env)
|
||
hash_value = make_hash(src_target)
|
||
file_info["hash"] = hash_value
|
||
return parsed_result
|
||
|
||
|
||
def apply_func(func: Callable[[str], str], value: list[str] | set[str] | str) -> list[str] | set[str] | str:
|
||
"""对一个字符串，或字符串序列，应用函数。"""
|
||
# 如：pkg_softlink列表
|
||
if isinstance(value, list):
|
||
return list(map(func, value))
|
||
# 如：feature集合
|
||
if isinstance(value, set):
|
||
return set(map(func, value))
|
||
return func(value)
|
||
|
||
|
||
REAL_PREFIX = "real:"
|
||
|
||
|
||
def join_dst_path(base: str, other: str) -> str:
|
||
"""联结dst_path。"""
|
||
if other.startswith("real:"):
|
||
other = other[len(REAL_PREFIX) :]
|
||
return other
|
||
return os.path.join(base, other)
|
||
|
||
|
||
def evaluate_info(info: dict[str, str], loaded_block: LoadedBlockElement, env_dict: EnvDict) -> dict[str, str]:
|
||
"""info元素求值。"""
|
||
dst_keys = ("dst_path",)
|
||
|
||
replace_env_func = partial(replace_env, env_dict)
|
||
add_dst_path_func = partial(join_dst_path, loaded_block.dst_path)
|
||
|
||
def upper_value(key: str, value: str) -> tuple[str, str]:
|
||
if key == "configurable":
|
||
return key, value.upper()
|
||
return key, value
|
||
|
||
def add_dst_path(key: str, value: str) -> tuple[str, str]:
|
||
if key in dst_keys:
|
||
return key, apply_func(add_dst_path_func, value)
|
||
return key, value
|
||
|
||
def replace_pkg_inner_softlink(key: str, value: str) -> tuple[str, str]:
|
||
if key == "pkg_inner_softlink":
|
||
return key, "NA"
|
||
return key, value
|
||
|
||
def merge_feature(key: str, value: str) -> tuple[str, str]:
|
||
if key in ("chip", "feature"):
|
||
config_features = config_feature_to_set(value, key)
|
||
return key, config_features | getattr(loaded_block, f"{key}s")
|
||
return key, value
|
||
|
||
def eval_value(_key: str, value: str) -> str:
|
||
if value is not None:
|
||
return apply_func(replace_env_func, value)
|
||
|
||
eval_value_func = star_pipe(
|
||
upper_value,
|
||
add_dst_path,
|
||
replace_pkg_inner_softlink,
|
||
merge_feature,
|
||
eval_value,
|
||
)
|
||
|
||
return {
|
||
key: eval_value_func(key, value)
|
||
for key, value in itertools.chain(
|
||
# 默认值配置
|
||
[
|
||
("dst_path", ""),
|
||
("configurable", "FALSE"),
|
||
("chip", None),
|
||
("feature", None),
|
||
("pkg_feature", None),
|
||
],
|
||
info.items(),
|
||
)
|
||
}
|
||
|
||
|
||
def parse_dir_info_elements(
|
||
loaded_block: LoadedBlockElement, default_config: dict[str, str], package_attr: PackageAttr, env: ParseEnv
|
||
) -> list[dict[str, str]]:
|
||
"""解析dir_info元素。"""
|
||
dir_info_elements: list[ET.Element] = loaded_block.root_ele.findall("dir_info")
|
||
dir_infos = []
|
||
for item in dir_info_elements:
|
||
dir_config = default_config.copy()
|
||
dir_config.update(item.attrib)
|
||
dir_config["module"] = dir_config.get("value")
|
||
for sub_item in list(item):
|
||
dir_info = dir_config.copy()
|
||
dir_info.update(sub_item.attrib)
|
||
dir_info = evaluate_info(dir_info, loaded_block, env.env_dict)
|
||
check_value(dir_info["value"], env.parse_option.package_check, package_attr)
|
||
dir_infos.append(dir_info)
|
||
|
||
return dir_infos
|
||
|
||
|
||
def expand_dir(file_info: FileInfo, get_dst_target_func: Callable[[FileInfo], str]):
|
||
"""
|
||
如果file_info中配置的路径是文件夹，需要展开到文件
|
||
"""
|
||
file_info_list = []
|
||
dir_info_list = []
|
||
dst_target = get_dst_target_func(file_info)
|
||
|
||
value_list = file_info.get("value").split("/")
|
||
target_name = value_list[-1] if value_list[-1] else value_list[-2]
|
||
|
||
# 这里把当前目录也加入到dir_info_list中
|
||
dir_info_copy = file_info.copy()
|
||
dir_info_copy["module"] = file_info.get("value")
|
||
dir_info_copy["value"] = os.path.join(file_info.get("install_path", ""), target_name)
|
||
|
||
# 子目录的权限按照xml中subdir_mod配置，如果没有配置subdir_mod按照install_mod配置
|
||
subdir_mod = file_info.get("subdir_mod", None)
|
||
if subdir_mod is not None:
|
||
dir_info_copy["install_mod"] = subdir_mod
|
||
# 被展开的当前目录不需要设置softlink
|
||
dir_info_copy["install_softlink"] = "NA"
|
||
dir_info_copy["pkg_inner_softlink"] = "NA"
|
||
dir_info_list.append(dir_info_copy)
|
||
|
||
for root, dirs, files in os.walk(dst_target, followlinks=True):
|
||
# 不同操作系统上，os.walk遍历的结果顺序会略有不同，这里按字母排序，保证不同系统一致
|
||
dirs.sort()
|
||
files.sort()
|
||
|
||
dirs_to_remove = []
|
||
for name in dirs:
|
||
dirname = os.path.join(root, name)
|
||
# 如果是指向目录的软连接，则按照文件处理，无需在安装时创建目录，只需要卸载时删除就行
|
||
if os.path.islink(dirname) and not need_dereference(file_info):
|
||
copy_file_info = create_file_info(dirname, dst_target, file_info, name, target_name)
|
||
# 被展开的子文件不需要设置softlink
|
||
copy_file_info["install_softlink"] = "NA"
|
||
copy_file_info["pkg_inner_softlink"] = "NA"
|
||
file_info_list.append(copy_file_info)
|
||
dirs_to_remove.append(name)
|
||
continue
|
||
relative_dirname = os.path.relpath(dirname, dst_target)
|
||
dir_info_copy = file_info.copy()
|
||
dir_info_copy["module"] = file_info.get("value")
|
||
dir_info_copy["value"] = os.path.join(file_info.get("install_path", ""), target_name, relative_dirname)
|
||
# 被展开的子目录不需要设置softlink
|
||
dir_info_copy["install_softlink"] = "NA"
|
||
dir_info_copy["pkg_inner_softlink"] = "NA"
|
||
# 子目录的权限按照xml中subdir_mod配置，如果没有配置subdir_mod按照install_mod配置
|
||
subdir_mod = file_info.get("subdir_mod", None)
|
||
if subdir_mod is not None:
|
||
dir_info_copy["install_mod"] = subdir_mod
|
||
dir_info_list.append(dir_info_copy)
|
||
for name in files:
|
||
filename = os.path.join(root, name)
|
||
copy_file_info = create_file_info(filename, dst_target, file_info, name, target_name)
|
||
file_info_list.append(copy_file_info)
|
||
|
||
for name in dirs_to_remove:
|
||
dirs.remove(name)
|
||
return file_info_list, dir_info_list
|
||
|
||
|
||
def create_file_info(dirname, dst_target, file_info, name, target_name):
|
||
relative_filename = os.path.relpath(dirname, dst_target)
|
||
relative_dir_name = os.path.split(relative_filename)[0]
|
||
copy_file_info = file_info.copy()
|
||
copy_file_info["value"] = name
|
||
copy_file_info["src_path"] = os.path.join(file_info["src_path"], file_info["value"], relative_dir_name)
|
||
copy_file_info["dst_path"] = os.path.join(file_info["dst_path"], target_name, relative_dir_name)
|
||
copy_file_info["install_path"] = os.path.join(file_info.get("install_path", ""), target_name, relative_dir_name)
|
||
return copy_file_info
|
||
|
||
|
||
def expand_file_info_asterisk(parsed_result: FileInfoParsedResult, env: ParseEnv) -> Iterator[FileInfoParsedResult]:
|
||
"""展开FileInfoParsedResult中的星号。"""
|
||
file_info = parsed_result.file_info
|
||
if check_contain_asterisk(file_info.get("value", "")):
|
||
dst_prefix = get_dst_prefix(file_info, env)
|
||
dst_targets = sorted(glob.glob(get_dst_target(file_info, env)))
|
||
if "exclude" in file_info:
|
||
exclude = list(map(methodcaller("strip"), file_info["exclude"].split(";")))
|
||
else:
|
||
exclude = []
|
||
for dst_target in dst_targets:
|
||
value = os.path.relpath(dst_target, dst_prefix)
|
||
if value in exclude:
|
||
continue
|
||
new_file_info = file_info.copy()
|
||
new_file_info["value"] = value
|
||
if "pkg_inner_softlink" in new_file_info:
|
||
# pkg_inner_softlink中的特殊变量$(FILE)替换为展开后的文件名
|
||
pkg_inner_softlink = new_file_info["pkg_inner_softlink"]
|
||
new_file_info["pkg_inner_softlink"] = pkg_inner_softlink.replace(
|
||
"$(FILE)", os.path.basename(dst_target)
|
||
)
|
||
yield parsed_result._replace(file_info=new_file_info)
|
||
else:
|
||
yield parsed_result
|
||
|
||
|
||
def trans_to_stream(item: Any) -> Iterator[Any]:
|
||
"""转换为流。"""
|
||
yield item
|
||
|
||
|
||
def need_dereference(file_info: FileInfo) -> bool:
|
||
"""是否需要解引用。"""
|
||
return "dereference" in file_info
|
||
|
||
|
||
def need_expand(file_info: FileInfo, get_dst_target_func: Callable[[FileInfo], str]) -> bool:
|
||
"""是否需要展开子目录。"""
|
||
if file_info.get("entity") == "true":
|
||
return False
|
||
dst_target = get_dst_target_func(file_info)
|
||
if os.path.isdir(dst_target):
|
||
if need_dereference(file_info):
|
||
return True
|
||
return not os.path.islink(dst_target)
|
||
return False
|
||
|
||
|
||
def expand_file_info(
|
||
parsed_result: FileInfoParsedResult, use_move: bool, get_dst_target_func: Callable[[FileInfo], str]
|
||
) -> FileInfoParsedResult:
|
||
"""展开FileInfoParsedResult中的目录。"""
|
||
file_info = parsed_result.file_info
|
||
if need_expand(file_info, get_dst_target_func):
|
||
# 如果当前是文件夹，需要展开计算
|
||
expand_infos, dir_infos = expand_dir(file_info, get_dst_target_func)
|
||
# 实测发现，对于opp包，整体目录cp的安装速度要快于目录中各文件mv
|
||
# 可能的原因是，cp遍历目录的速度较快，并且目录中的文件都比较小。mv依赖shell迭代目录中的所有文件。
|
||
return FileInfoParsedResult(merge_dict(file_info, {"is_dir": True}), [], dir_infos, expand_infos)
|
||
|
||
if use_move:
|
||
return FileInfoParsedResult({}, [file_info], parsed_result.dir_infos, parsed_result.expand_infos)
|
||
|
||
return parsed_result
|
||
|
||
|
||
def trans_file_info_to_result(file_info: FileInfo) -> FileInfoParsedResult:
|
||
"""file_info转换为FileInfoParsedResult。"""
|
||
return FileInfoParsedResult(file_info, [], [], [])
|
||
|
||
|
||
def parse_file_element(
|
||
file_ele: ET.Element,
|
||
file_config: dict[str, str],
|
||
loaded_block: LoadedBlockElement,
|
||
package_attr: PackageAttr,
|
||
env: ParseEnv,
|
||
) -> Iterator[FileInfoParsedResult]:
|
||
"""解析file元素。"""
|
||
file_info = merge_dict(file_config, file_ele.attrib)
|
||
file_info = evaluate_info(file_info, loaded_block, env.env_dict)
|
||
|
||
if package_attr.get("expand_asterisk", False):
|
||
expand_asterisk_func = partial(expand_file_info_asterisk, env=env)
|
||
else:
|
||
expand_asterisk_func = trans_to_stream
|
||
|
||
if "install_path" not in file_info:
|
||
file_info["install_path"] = ""
|
||
|
||
trans_file_info_func = pipe(
|
||
trans_file_info_to_result,
|
||
expand_asterisk_func,
|
||
partial(map, partial(config_hash, env=env)),
|
||
partial(
|
||
map,
|
||
partial(
|
||
expand_file_info, use_move=loaded_block.use_move, get_dst_target_func=partial(get_dst_target, env=env)
|
||
),
|
||
),
|
||
)
|
||
|
||
yield from trans_file_info_func(file_info)
|
||
|
||
|
||
def parse_file_info_elements(
|
||
loaded_block: LoadedBlockElement, default_config: dict[str, str], package_attr: PackageAttr, env: ParseEnv
|
||
) -> Iterator[FileInfoParsedResult]:
|
||
"""解析file_info元素。"""
|
||
file_info_elements: list[ET.Element] = loaded_block.root_ele.findall("file_info")
|
||
for file_info_ele in file_info_elements:
|
||
file_config = merge_dict(default_config, file_info_ele.attrib, {"module": file_info_ele.attrib.get("value")})
|
||
|
||
for sub_item in list(file_info_ele):
|
||
yield from parse_file_element(sub_item, file_config, loaded_block, package_attr, env)
|
||
|
||
|
||
def unique_infos(infos: Iterable) -> list[dict[str, str]]:
|
||
"""infos去重。"""
|
||
cache: set[str] = set()
|
||
new_infos = []
|
||
for info in infos:
|
||
if info["value"] in cache:
|
||
continue
|
||
cache.add(info["value"])
|
||
new_infos.append(info)
|
||
|
||
return new_infos
|
||
|
||
|
||
def parse_block_config(loaded_block: LoadedBlockElement, package_attr: PackageAttr, parse_env: ParseEnv):
|
||
"""解析块配置。"""
|
||
default_config = copy.copy(loaded_block.attrs)
|
||
default_config.update(loaded_block.root_ele.attrib)
|
||
|
||
dir_infos = parse_dir_info_elements(
|
||
loaded_block,
|
||
default_config,
|
||
package_attr,
|
||
parse_env,
|
||
)
|
||
file_info_results = list(
|
||
chain(
|
||
parse_file_info_elements(
|
||
loaded_block,
|
||
default_config,
|
||
package_attr,
|
||
parse_env,
|
||
)
|
||
)
|
||
)
|
||
|
||
generate_infos = parse_generate_infos_by_loaded_block(loaded_block, default_config, parse_env.env_dict)
|
||
|
||
return BlockConfig(
|
||
unique_infos(itertools.chain(dir_infos, flatten(result.dir_infos for result in file_info_results))),
|
||
list(flatten(map(attrgetter("move_infos"), file_info_results))),
|
||
list(flatten(map(attrgetter("expand_infos"), file_info_results))),
|
||
[result.file_info for result in file_info_results if result.file_info],
|
||
generate_infos,
|
||
)
|
||
|
||
|
||
def make_loaded_block_element(root_ele: ET.Element, dst_path: str = "") -> LoadedBlockElement:
|
||
"""创建加载后的块配置。"""
|
||
return LoadedBlockElement(root_ele, False, dst_path, set(), set(), {})
|
||
|
||
|
||
def parse_block_element(block_ele: ET.Element, block_info_attr: dict[str, str]) -> BlockElement:
|
||
"""解析单个块配置。"""
|
||
|
||
def filter_attrs(attrs: dict[str, str]) -> dict[str, str]:
|
||
# block属性中过滤掉dst_path与block_conf_path
|
||
# dst_path由单独的参数传递
|
||
# block中不需要block_conf_path
|
||
return {key: value for key, value in attrs.items() if key not in ("dst_path", "block_conf_path")}
|
||
|
||
def with_merged_attrs(attrs: dict[str, str]) -> BlockElement:
|
||
name = attrs.get("name")
|
||
block_conf_path = attrs.get("block_conf_path")
|
||
|
||
if not name:
|
||
raise BlockConfigError("block's name is not set!")
|
||
|
||
if not block_conf_path:
|
||
raise BlockConfigError("block's conf_path is not set!")
|
||
|
||
return BlockElement(
|
||
name=name,
|
||
block_conf_path=block_conf_path,
|
||
dst_path=attrs.get("dst_path", ""),
|
||
chips=config_feature_to_set(attrs.get("chip"), "chip"),
|
||
features=config_feature_to_set(attrs.get("feature"), "feature"),
|
||
attrs=filter_attrs(attrs),
|
||
)
|
||
|
||
return with_merged_attrs(merge_dict(block_info_attr, block_ele.attrib))
|
||
|
||
|
||
def parse_block_info(block_info: ET.Element) -> list[BlockElement]:
|
||
"""解析块配置。"""
|
||
|
||
def parse_block_elements(block_elements: list[ET.Element]) -> list[BlockElement]:
|
||
return [parse_block_element(block_ele, block_info.attrib) for block_ele in block_elements]
|
||
|
||
return parse_block_elements(list(block_info))
|
||
|
||
|
||
def get_block_filepath(block_element: BlockElement) -> str:
|
||
"""获取块配置路径。"""
|
||
return os.path.join(
|
||
pkg_utils.TOP_SOURCE_DIR, BLOCK_CONFIG_PATH, block_element.block_conf_path, f"{block_element.name}.xml"
|
||
)
|
||
|
||
|
||
def load_block_element(package_attr: PackageAttr, block_element: BlockElement) -> LoadedBlockElement:
|
||
"""加载块配置。"""
|
||
|
||
def with_filepath(block_xml: str):
|
||
if not os.path.exists(block_xml):
|
||
raise BlockConfigError(f"block's config xml {block_xml} does not exist!")
|
||
|
||
try:
|
||
return LoadedBlockElement(
|
||
root_ele=ET.parse(block_xml).getroot(),
|
||
use_move=package_attr.get("use_move", False),
|
||
**{name: getattr(block_element, name) for name in BLOCK_ELEMENT_PASS_THROUGH_ARGS},
|
||
)
|
||
except Exception:
|
||
raise BlockConfigError(f"dependent block configuration {block_xml} parse failed!")
|
||
|
||
return with_filepath(get_block_filepath(block_element))
|
||
|
||
|
||
def parse_blocks(root_ele: ET.Element, package_attr: PackageAttr, parse_env: ParseEnv) -> list[BlockConfig]:
|
||
"""解析块列表。"""
|
||
return [
|
||
parse_block_config(loaded_block, package_attr, parse_env)
|
||
for loaded_block in itertools.chain(
|
||
[make_loaded_block_element(root_ele)],
|
||
map(
|
||
partial(load_block_element, package_attr),
|
||
chain.from_iterable(map(parse_block_info, root_ele.findall("block_info"))),
|
||
),
|
||
)
|
||
]
|
||
|
||
|
||
def read_version_info() -> tuple[str, str]:
|
||
version_path = os.path.join(pkg_utils.TOP_DIR, "version.info")
|
||
with open(version_path) as file:
|
||
line1 = file.readline().strip()
|
||
line2 = file.readline().strip()
|
||
version = line1.split("=")[1]
|
||
version_dir = line2.split("=")[1]
|
||
m = re.match(r"[.a-zA-Z0-9]+$", version) or re.match(r"[-a-zA-Z.0-9]+$", version)
|
||
if not m:
|
||
raise VersionFormatNotMatch()
|
||
|
||
return version, version_dir
|
||
|
||
|
||
def parse_xml_config(filepath: str, delivery_dir: str, parse_option: ParseOption, args: Namespace) -> XmlConfig:
|
||
"""解析打包xml配置。"""
|
||
try:
|
||
tree = ET.parse(filepath)
|
||
xml_root = tree.getroot()
|
||
except ET.ParseError as ex:
|
||
CommLog.cilog_error("xml parse %s failed: %s!", filepath, ex)
|
||
sys.exit(FAIL)
|
||
|
||
default_config = xml_root.attrib.copy()
|
||
|
||
package_attr = parse_package_attr(xml_root, args)
|
||
if args.version_dir:
|
||
version = args.version_dir
|
||
version_dir = args.version_dir
|
||
else:
|
||
version, version_dir = read_version_info()
|
||
if args.disable_multi_version:
|
||
version_dir = None
|
||
timestamp = get_timestamp(args)
|
||
try:
|
||
env_dict = parse_env_dict(parse_option.os_arch, package_attr, version, version_dir, timestamp)
|
||
except ParseOsArchError:
|
||
CommLog.cilog_error("os_arch %s is not correctly configured: %s!", parse_option.os_arch, filepath)
|
||
sys.exit(FAIL)
|
||
|
||
parse_env = ParseEnv(env_dict, parse_option, delivery_dir, pkg_utils.TOP_SOURCE_DIR)
|
||
|
||
blocks = parse_blocks(xml_root, package_attr, parse_env)
|
||
|
||
if is_multi_version(version_dir):
|
||
fill_is_common_path_func = partial(fill_is_common_path, target_env=env_dict.get("TARGET_ENV"))
|
||
else:
|
||
fill_is_common_path_func = iter
|
||
|
||
return XmlConfig(default_config, package_attr, None, blocks, version, None, PackerConfig(fill_is_common_path_func))
|