516 lines
16 KiB
Python
516 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: UTF-8 -*-
|
||
# -----------------------------------------------------------------------------------------------------------
|
||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||
# See LICENSE in the root of the software repository for the full text of the License.
|
||
# -----------------------------------------------------------------------------------------------------------
|
||
|
||
"""filelist相关类。"""
|
||
|
||
import itertools
|
||
import os
|
||
from collections import Counter
|
||
from collections.abc import Callable, Iterator
|
||
from enum import IntEnum
|
||
from functools import partial
|
||
from itertools import chain, repeat
|
||
from operator import and_, attrgetter, contains, itemgetter, lt, methodcaller, ne, not_
|
||
from typing import NamedTuple
|
||
|
||
from .utils.comm_log import CommLog
|
||
from .utils.funcbase import any_, constant, dispatch, identity, invoke, pipe, side_effect, star_apply
|
||
from .utils.pkg_utils import (
|
||
TOP_DIR,
|
||
FilelistError,
|
||
GenerateFilelistError,
|
||
conditional_apply,
|
||
config_feature_to_string,
|
||
pairwise,
|
||
swap_args,
|
||
)
|
||
|
||
|
||
class FileItem(NamedTuple):
|
||
"""文件条目"""
|
||
|
||
module: str
|
||
operation: str
|
||
relative_path_in_pkg: str
|
||
relative_install_path: str
|
||
is_in_docker: str
|
||
permission: str
|
||
owner_group: str
|
||
install_type: str
|
||
softlink: list[str]
|
||
feature: set[str]
|
||
is_common_path: str
|
||
configurable: str
|
||
hash_value: str
|
||
block: str
|
||
pkg_inner_softlink: list[str]
|
||
chip: set[str]
|
||
is_dir: bool
|
||
|
||
|
||
def create_file_item(*args, **kwargs) -> FileItem:
|
||
"""创建文件条目。"""
|
||
file_item = FileItem(*args, **kwargs)
|
||
|
||
if not isinstance(file_item.feature, set):
|
||
raise TypeError("The feature parameter should be a set.")
|
||
if not isinstance(file_item.chip, set):
|
||
raise TypeError("The chip parameter should be a set.")
|
||
if not isinstance(file_item.softlink, list):
|
||
raise TypeError("The softlink parameter should be a list.")
|
||
if not isinstance(file_item.pkg_inner_softlink, list):
|
||
raise TypeError("The pkg_inner_softlink parameter should be a list.")
|
||
|
||
return file_item
|
||
|
||
|
||
# 文件列表
|
||
FileList = list[FileItem]
|
||
|
||
|
||
def soft_links_to_string(soft_links: list[str]) -> str:
|
||
"""软链接转换为字符串。"""
|
||
if not soft_links:
|
||
return "NA"
|
||
return ";".join(soft_links)
|
||
|
||
|
||
def file_item_to_string(item: FileItem) -> str:
|
||
"""文件条目转换为字符串。"""
|
||
return ",".join(
|
||
[
|
||
item.module,
|
||
item.operation,
|
||
item.relative_path_in_pkg,
|
||
item.relative_install_path,
|
||
item.is_in_docker,
|
||
item.permission,
|
||
item.owner_group,
|
||
item.install_type,
|
||
soft_links_to_string(item.softlink),
|
||
config_feature_to_string(item.feature),
|
||
item.is_common_path,
|
||
item.configurable,
|
||
item.hash_value,
|
||
item.block,
|
||
soft_links_to_string(item.pkg_inner_softlink),
|
||
config_feature_to_string(item.chip),
|
||
]
|
||
)
|
||
|
||
|
||
def get_filelist_header_string() -> str:
|
||
"""获取文件列表表头。"""
|
||
return ",".join(
|
||
[
|
||
"module",
|
||
"operation",
|
||
"relative_path_in_pkg",
|
||
"relative_install_path",
|
||
"is_in_docker",
|
||
"permission",
|
||
"owner:group",
|
||
"install_type",
|
||
"softlink",
|
||
"feature",
|
||
"is_common_path",
|
||
"configurable",
|
||
"hash",
|
||
"block",
|
||
"pkg_inner_softlink",
|
||
"chip",
|
||
]
|
||
)
|
||
|
||
|
||
def get_soft_links_not_in_common_paths(filelist: FileList, target_env: str) -> Iterator[list[str]]:
|
||
for file_item_t in filelist:
|
||
if file_item_t.relative_install_path.startswith(target_env):
|
||
for softlink in file_item_t.softlink:
|
||
if not softlink.startswith(target_env):
|
||
yield softlink
|
||
|
||
|
||
def fill_is_common_path(filelist: FileList, target_env: str) -> Iterator[FileItem]:
|
||
"""填充文件条目中是否为公共目录字段。"""
|
||
soft_links = set(get_soft_links_not_in_common_paths(filelist, target_env))
|
||
for file_item in filelist:
|
||
if file_item.relative_install_path.startswith(target_env):
|
||
yield file_item._replace(is_common_path="Y")
|
||
else:
|
||
is_soft_links_prefix = map(methodcaller("startswith", f"{file_item.relative_install_path}/"), soft_links)
|
||
if any(is_soft_links_prefix):
|
||
yield file_item._replace(is_common_path="YY")
|
||
else:
|
||
yield file_item
|
||
|
||
|
||
def is_relative_install_path(path: str) -> bool:
|
||
"""是否为相对路径。"""
|
||
return not path.startswith("/")
|
||
|
||
|
||
def is_specific_operations(file_item: FileItem, operations: list[str]) -> bool:
|
||
"""是否为特定的操作类型。"""
|
||
return file_item.operation in operations
|
||
|
||
|
||
def is_specific_install_type(file_item: FileItem, install_types: set[str]) -> bool:
|
||
"""是否为特定的安装类型。"""
|
||
item_install_types = set(file_item.install_type.split(";"))
|
||
if "all" in item_install_types:
|
||
return True
|
||
return bool(item_install_types & install_types)
|
||
|
||
|
||
def get_install_path_dirs(install_path: str) -> Iterator[str]:
|
||
"""获取安装路径父目录。"""
|
||
install_path = os.path.dirname(install_path)
|
||
while install_path not in ("", "/"):
|
||
yield install_path
|
||
install_path = os.path.dirname(install_path)
|
||
|
||
|
||
def get_missing_dir_set(filelist: FileList) -> set[str]:
|
||
"""获取缺失目录集合。
|
||
|
||
文件列表可能出现某一级目录缺失情况。
|
||
如配置了file_info:aaa/bbb/ccc.txt，但只配置了dir_info:aaa，
|
||
那么缺失dir_info:aaa/bbb
|
||
"""
|
||
parent_dirs: set[str] = invoke(
|
||
pipe(
|
||
dispatch(
|
||
pipe(
|
||
partial(
|
||
filter,
|
||
partial(is_specific_operations, operations={"copy", "copy_entity"}),
|
||
),
|
||
partial(map, attrgetter("relative_install_path")),
|
||
partial(filter, is_relative_install_path),
|
||
set,
|
||
partial(map, get_install_path_dirs),
|
||
chain.from_iterable,
|
||
),
|
||
pipe(
|
||
partial(map, attrgetter("softlink")),
|
||
chain.from_iterable,
|
||
partial(
|
||
filter,
|
||
pipe(
|
||
dispatch(
|
||
bool,
|
||
is_relative_install_path,
|
||
partial(ne, "NA"),
|
||
),
|
||
all,
|
||
),
|
||
),
|
||
set,
|
||
partial(map, get_install_path_dirs),
|
||
chain.from_iterable,
|
||
),
|
||
pipe(
|
||
partial(map, attrgetter("pkg_inner_softlink")),
|
||
chain.from_iterable,
|
||
partial(
|
||
filter,
|
||
pipe(
|
||
dispatch(
|
||
bool,
|
||
partial(ne, "NA"),
|
||
),
|
||
all,
|
||
),
|
||
),
|
||
set,
|
||
partial(map, get_install_path_dirs),
|
||
chain.from_iterable,
|
||
),
|
||
),
|
||
chain.from_iterable,
|
||
set,
|
||
),
|
||
filelist,
|
||
)
|
||
mkdir_installs: set[str] = {
|
||
file_item.relative_install_path
|
||
for file_item in filter(partial(is_specific_operations, operations={"mkdir"}), filelist)
|
||
if is_relative_install_path(file_item.relative_install_path)
|
||
}
|
||
|
||
mkdir_parent_dirs: set[str] = set(itertools.chain.from_iterable(map(get_install_path_dirs, mkdir_installs)))
|
||
|
||
missing_dir_set = sorted((parent_dirs | mkdir_parent_dirs) - mkdir_installs)
|
||
return set(missing_dir_set)
|
||
|
||
|
||
def print_missing_dir_set(missing_dir_set: set[str], in_msg: str = None) -> set[str]:
|
||
"""打印缺失目录集合。"""
|
||
if in_msg:
|
||
tail_msg = f" {in_msg}"
|
||
else:
|
||
tail_msg = ""
|
||
for path in sorted(missing_dir_set):
|
||
CommLog.cilog_error(f'missing dir info path "{path}"{tail_msg}')
|
||
return missing_dir_set
|
||
|
||
|
||
def print_unsafe_paths(unsafe_paths: tuple[str, ...]) -> tuple[str, ...]:
|
||
"""打印非安全路径。"""
|
||
for path in unsafe_paths:
|
||
CommLog.cilog_error(f'unsafe path "{path}" in move scene.')
|
||
return unsafe_paths
|
||
|
||
|
||
# 获取filelist中所有的特性集合
|
||
get_features_in_filelist = pipe(
|
||
partial(map, attrgetter("feature")),
|
||
chain.from_iterable, # 展开集合序列为元素序列
|
||
set, # 去重
|
||
partial(filter, partial(ne, "comm")), # 排除comm特性
|
||
set,
|
||
)
|
||
|
||
# 获取filelist中所有的芯片集合
|
||
get_chips_in_filelist = pipe(
|
||
partial(map, attrgetter("chip")),
|
||
chain.from_iterable, # 展开集合序列为元素序列
|
||
set, # 去重
|
||
)
|
||
|
||
|
||
def check_features_in_filelist(features: set[str], filelist: FileList) -> set[str]:
|
||
"""检查文件列表中特性配置目录规范。"""
|
||
return invoke(
|
||
pipe(
|
||
# 过滤指定features的file_item
|
||
partial(filter, pipe(attrgetter("feature"), partial(and_, features), bool)),
|
||
list,
|
||
get_missing_dir_set,
|
||
partial(print_missing_dir_set, in_msg=f"in features {features}"),
|
||
),
|
||
filelist,
|
||
)
|
||
|
||
|
||
def check_chip_in_filelist(chip: str, filelist: FileList) -> set[str]:
|
||
"""检查文件列表中芯片配置目录规范。"""
|
||
return invoke(
|
||
pipe(
|
||
# 过滤指定chip的file_item
|
||
partial(
|
||
filter,
|
||
any_(
|
||
pipe(attrgetter("chip"), not_), # 没有配置chip
|
||
pipe(attrgetter("chip"), partial(swap_args(contains), chip), bool), # 配置了指定chip
|
||
),
|
||
),
|
||
list,
|
||
get_missing_dir_set,
|
||
partial(print_missing_dir_set, in_msg=f"in chip {chip}"),
|
||
),
|
||
filelist,
|
||
)
|
||
|
||
|
||
check_filelist_features = any_(
|
||
pipe(
|
||
dispatch(
|
||
pipe(
|
||
get_features_in_filelist,
|
||
# 对于每个feature，与comm组成一个set
|
||
partial(map, lambda x: {x, "comm"}),
|
||
# 此时为feature集合序列
|
||
),
|
||
repeat, # 重复filelist
|
||
),
|
||
tuple,
|
||
star_apply(zip),
|
||
# 此时为元组序列，元组的第1个元素是过滤的feature集合，第2个元素是filelist
|
||
partial(itertools.starmap, check_features_in_filelist),
|
||
# 此时为集合序列；合并为一个集合
|
||
chain.from_iterable,
|
||
set,
|
||
),
|
||
pipe(
|
||
dispatch(
|
||
get_chips_in_filelist,
|
||
repeat, # 重复filelist
|
||
),
|
||
tuple,
|
||
star_apply(zip),
|
||
# 此时为元组序列，元组的第1个元素是chip集合，第2个元素是filelist
|
||
partial(itertools.starmap, check_chip_in_filelist),
|
||
# 此时为集合序列；合并为一个集合
|
||
chain.from_iterable,
|
||
set,
|
||
),
|
||
)
|
||
|
||
|
||
# 检查move是否安全，是否存在同一个源路径被mv多次
|
||
check_move_safe = pipe(
|
||
partial(
|
||
filter,
|
||
partial(is_specific_operations, operations={"copy", "copy_entity", "move"}),
|
||
),
|
||
partial(map, attrgetter("relative_path_in_pkg")),
|
||
Counter,
|
||
methodcaller("items"),
|
||
partial(filter, pipe(itemgetter(1), partial(lt, 1))),
|
||
partial(map, itemgetter(0)),
|
||
tuple,
|
||
print_unsafe_paths,
|
||
)
|
||
|
||
|
||
def check_filelist(filelist: FileList, check_features: bool, check_move: bool):
|
||
"""检查文件列表是否符合规范。"""
|
||
if check_features:
|
||
check_features_func = check_filelist_features
|
||
else:
|
||
check_features_func = constant(set())
|
||
|
||
if check_move:
|
||
check_move_func = check_move_safe
|
||
else:
|
||
check_move_func = constant(tuple())
|
||
|
||
# 此处使用any_，短路部分报错
|
||
check_func = any_(
|
||
pipe(
|
||
get_missing_dir_set,
|
||
print_missing_dir_set,
|
||
),
|
||
pipe(
|
||
partial(filter, partial(is_specific_install_type, install_types={"run"})),
|
||
list,
|
||
get_missing_dir_set,
|
||
partial(print_missing_dir_set, in_msg="in run install type"),
|
||
),
|
||
check_features_func,
|
||
check_move_func,
|
||
)
|
||
missing = check_func(filelist)
|
||
|
||
if missing:
|
||
raise FilelistError()
|
||
|
||
|
||
def get_common_path(args: list[str]) -> str:
|
||
"""公共路径前缀。"""
|
||
try:
|
||
return os.path.commonpath(args)
|
||
except ValueError:
|
||
return ""
|
||
|
||
|
||
class FileItemRelation(IntEnum):
|
||
"""文件条目之间的关系。"""
|
||
|
||
NOT_NESTED = 0 # 不是嵌套文件
|
||
NESTED = 1 # 嵌套文件
|
||
SAME = 2 # 相同文件
|
||
|
||
|
||
def is_nested_file_item(item: FileItem, base_item: FileItem) -> FileItemRelation:
|
||
"""是否为嵌套的文件。"""
|
||
if base_item is None:
|
||
return FileItemRelation.NOT_NESTED
|
||
|
||
if item == base_item:
|
||
return FileItemRelation.SAME
|
||
|
||
install_path = item.relative_install_path
|
||
base_install_path = base_item.relative_install_path
|
||
|
||
common_install_path = get_common_path([install_path, base_install_path])
|
||
if common_install_path != base_install_path:
|
||
return FileItemRelation.NOT_NESTED
|
||
|
||
pkg_path = item.relative_path_in_pkg
|
||
base_pkg_path = base_item.relative_path_in_pkg
|
||
|
||
install_rel_path = os.path.relpath(install_path, base_install_path)
|
||
pkg_rel_path = os.path.relpath(pkg_path, base_pkg_path)
|
||
if install_rel_path != pkg_rel_path:
|
||
# 确保打包与安装相对路径一致
|
||
raise FilelistError(f"nested paths {item} and {base_item} are illegal.")
|
||
return FileItemRelation.NESTED
|
||
|
||
|
||
def found_nested_file_item(item: FileItem, base_item: FileItem):
|
||
"""发现嵌套元素。"""
|
||
raise FilelistError(f"found nested paths {item} and {base_item}!")
|
||
|
||
|
||
def convert_nested_path_in_filelist(filelist: FileList):
|
||
"""filelist中嵌套路径元素转为del。"""
|
||
pre_item = None
|
||
for item in filelist:
|
||
ret = is_nested_file_item(item, pre_item)
|
||
if ret == FileItemRelation.NESTED:
|
||
yield item._replace(operation="del")
|
||
elif any((ret == FileItemRelation.NOT_NESTED, (ret == FileItemRelation.SAME and not item.is_dir))):
|
||
yield item
|
||
pre_item = item
|
||
|
||
|
||
# 检查文件列表中的嵌套路径。入参: filelist
|
||
check_nested_path_in_filelist = pipe(
|
||
partial(filter, partial(is_specific_operations, operations={"copy", "copy_entity"})),
|
||
partial(sorted, key=attrgetter("relative_install_path")),
|
||
pairwise,
|
||
partial(map, conditional_apply(star_apply(is_nested_file_item), star_apply(found_nested_file_item))),
|
||
list,
|
||
)
|
||
|
||
|
||
# 变换文件列表中嵌套路径。入参: filelist
|
||
transform_nested_path_in_filelist = pipe(
|
||
dispatch(
|
||
partial(itertools.filterfalse, partial(is_specific_operations, operations={"copy"})),
|
||
pipe(
|
||
partial(filter, partial(is_specific_operations, operations={"copy"})),
|
||
partial(sorted, key=attrgetter("relative_install_path")),
|
||
convert_nested_path_in_filelist,
|
||
),
|
||
),
|
||
chain.from_iterable,
|
||
list,
|
||
side_effect(check_nested_path_in_filelist),
|
||
)
|
||
|
||
|
||
def generate_filelist(filelist: FileList, filename: str):
|
||
"""生成文件列表文件。"""
|
||
content_list = list(
|
||
itertools.chain([get_filelist_header_string()], [file_item_to_string(item) for item in filelist])
|
||
)
|
||
content = "\n".join(content_list)
|
||
filepath = os.path.join(TOP_DIR, "build", filename)
|
||
try:
|
||
with open(filepath, "w", encoding="utf-8") as file:
|
||
file.write(content)
|
||
# filelist.csv文件末尾补充一个换行符
|
||
file.write("\n")
|
||
except OSError as ex:
|
||
raise GenerateFilelistError(filename) from ex
|
||
|
||
|
||
def get_transform_nested_path_func(parallel: bool) -> Callable[[FileList], FileList]:
|
||
"""获取转换嵌套路径函数。"""
|
||
if parallel:
|
||
return transform_nested_path_in_filelist
|
||
return identity
|