first commit
This commit is contained in:
185
dev/tools/completion_check.py
Normal file
185
dev/tools/completion_check.py
Normal file
@@ -0,0 +1,185 @@
|
||||
import json
|
||||
import re
|
||||
|
||||
from collections import Counter
|
||||
|
||||
|
||||
def should_ignore(weight_name, exact_ignore: list, regex_ignore: list):
|
||||
if any(weight_name.startswith(ig) for ig in exact_ignore):
|
||||
return True
|
||||
if any(p.match(weight_name) for p in regex_ignore):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def compile_ignore(ignore_list: list | None = None,
|
||||
extra_ignore_list: list | None = None):
|
||||
exact = set()
|
||||
regex = []
|
||||
|
||||
for x in (ignore_list or []) + (extra_ignore_list or []):
|
||||
if x.startswith("re:"):
|
||||
regex.append(re.compile(x[3:]))
|
||||
else:
|
||||
exact.add(x)
|
||||
return exact, regex
|
||||
|
||||
|
||||
def normalize_to_regex(weight_name: str) -> str:
|
||||
parts = []
|
||||
for p in weight_name.split("."):
|
||||
if p.isdigit():
|
||||
parts.append(r"\d+")
|
||||
else:
|
||||
parts.append(p)
|
||||
return "re:" + ".".join(parts) + ".*"
|
||||
|
||||
|
||||
def infer_default_quant_target(quant_config: dict) -> str:
|
||||
classes = quant_config.get("class") or {}
|
||||
int4_list = classes.get("int4") or []
|
||||
int8_list = classes.get("int8") or []
|
||||
|
||||
if int4_list and not int8_list:
|
||||
return "int4"
|
||||
if int8_list and not int4_list:
|
||||
return "int8"
|
||||
if len(int4_list) >= len(int8_list):
|
||||
return "int4"
|
||||
return "int8"
|
||||
|
||||
|
||||
def suggest_category(weight_name: str, default_quant_target: str) -> tuple[str, str]:
|
||||
wn = weight_name.lower()
|
||||
|
||||
ignore_keywords = [
|
||||
"lm_head",
|
||||
"embed_tokens",
|
||||
".input_layernorm.",
|
||||
".post_attention_layernorm.",
|
||||
".self_attn.k_norm.",
|
||||
".self_attn.q_norm.",
|
||||
".norm.",
|
||||
".bias",
|
||||
".e_score_correction_bias",
|
||||
]
|
||||
if any(k in wn for k in ignore_keywords):
|
||||
return "ignore", "norm/embed/bias 类型，通常放在 ignore"
|
||||
|
||||
quant_keywords = [
|
||||
".self_attn.",
|
||||
".mlp.",
|
||||
".experts.",
|
||||
"_proj.",
|
||||
".proj.",
|
||||
".linear.",
|
||||
]
|
||||
if any(k in wn for k in quant_keywords):
|
||||
return default_quant_target, f"线性/投影权重，建议放在 {default_quant_target}"
|
||||
|
||||
return "manual-review", "命名不典型，建议人工确认后归类"
|
||||
|
||||
|
||||
def collect_quant_ignore_list(quant_config: dict) -> list[str]:
|
||||
classes = quant_config.get("class") or {}
|
||||
ignore_list = []
|
||||
ignore_list += quant_config.get("ignore") or []
|
||||
ignore_list += classes.get("int4") or []
|
||||
ignore_list += classes.get("int8") or []
|
||||
return ignore_list
|
||||
|
||||
|
||||
def build_missing_summary(missing: list[str], quant_config: dict, max_show: int) -> dict:
|
||||
default_quant_target = infer_default_quant_target(quant_config)
|
||||
category_counter = Counter()
|
||||
rows = []
|
||||
|
||||
for wn in missing:
|
||||
category, reason = suggest_category(wn, default_quant_target)
|
||||
category_counter[category] += 1
|
||||
rows.append(
|
||||
{
|
||||
"weight_name": wn,
|
||||
"suggested_category": category,
|
||||
"suggested_pattern": normalize_to_regex(wn),
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"missing_total": len(missing),
|
||||
"default_quant_target": default_quant_target,
|
||||
"category_summary": {
|
||||
"ignore": category_counter.get("ignore", 0),
|
||||
"int4": category_counter.get("int4", 0),
|
||||
"int8": category_counter.get("int8", 0),
|
||||
"manual-review": category_counter.get("manual-review", 0),
|
||||
},
|
||||
"top_missing": rows[:max_show],
|
||||
}
|
||||
|
||||
|
||||
def summarize_missing(missing: list[str], quant_config: dict, max_show: int):
|
||||
summary = build_missing_summary(missing=missing, quant_config=quant_config, max_show=max_show)
|
||||
print(f"Missing total: {len(missing)}")
|
||||
print(f"Default quant target inferred from config: {summary['default_quant_target']}")
|
||||
print("Suggested category summary:")
|
||||
for cat, count in summary["category_summary"].items():
|
||||
print(f" - {cat}: {count}")
|
||||
|
||||
if not missing:
|
||||
print("All weights are covered by quant.ignore/int4/int8.")
|
||||
return
|
||||
|
||||
print(f"\nTop {len(summary['top_missing'])} missing items:")
|
||||
for idx, row in enumerate(summary["top_missing"], start=1):
|
||||
print(f"[{idx}] {row['weight_name']}")
|
||||
print(f" suggestion: {row['suggested_category']}")
|
||||
print(f" reason: {row['reason']}")
|
||||
print(f" pattern: {row['suggested_pattern']}")
|
||||
|
||||
|
||||
def check(weight_names, ignore_list):
|
||||
exact_ignore, regex_ignore = compile_ignore(ignore_list)
|
||||
missing = []
|
||||
for wn in weight_names:
|
||||
if not should_ignore(wn, exact_ignore, regex_ignore):
|
||||
missing.append(wn)
|
||||
return missing
|
||||
|
||||
|
||||
def build_validation_result(
|
||||
weight_names: list[str], quant_config: dict, max_show: int = 20
|
||||
) -> dict:
|
||||
ignore_list = collect_quant_ignore_list(quant_config)
|
||||
missing = check(weight_names, ignore_list)
|
||||
return build_missing_summary(missing=missing, quant_config=quant_config, max_show=max_show)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import yaml
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("--weight-map", type=str)
|
||||
parser.add_argument("--model-info", type=str)
|
||||
parser.add_argument("--max-show", type=int, default=20)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.model_info, "r") as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
quant_config = cfg['quant']
|
||||
ignore_list = collect_quant_ignore_list(quant_config)
|
||||
|
||||
with open(args.weight_map, "r") as f:
|
||||
weight_names = json.load(f)['weight_map'].keys()
|
||||
|
||||
missing = check(list(weight_names), ignore_list)
|
||||
summarize_missing(missing, quant_config, args.max_show)
|
||||
|
||||
|
||||
|
||||
78
dev/tools/filter_weight_map.py
Normal file
78
dev/tools/filter_weight_map.py
Normal file
@@ -0,0 +1,78 @@
|
||||
# 过滤明确量化/非量化的layer_name
|
||||
import re
|
||||
import os
|
||||
import json
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def filter(input_dir: str, filter_regex_list: list | None = None):
|
||||
|
||||
if filter_regex_list is None:
|
||||
filter_regex_list = []
|
||||
|
||||
candidate_suffixes = [
|
||||
"model.safetensors.index.json",
|
||||
"consolidated.safetensors.index.json",
|
||||
]
|
||||
|
||||
for cs in candidate_suffixes:
|
||||
input_index_path = os.path.join(input_dir, cs)
|
||||
if os.path.exists(input_index_path):
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"Can not find index file, {input_dir}")
|
||||
|
||||
with open(input_index_path, "r") as f:
|
||||
weight_map = json.load(f)['weight_map']
|
||||
|
||||
deduped = deduplicate_weight_map(weight_map)
|
||||
|
||||
kept_weight_names = {}
|
||||
|
||||
for w, sf in deduped.items():
|
||||
if any(re.match(p, w) for p in filter_regex_list):
|
||||
continue
|
||||
kept_weight_names[w] = sf
|
||||
|
||||
with open("temp.json", "w") as f:
|
||||
json.dump(kept_weight_names, f, indent=2)
|
||||
|
||||
|
||||
def structure_key(weight_name: str) -> str:
|
||||
"""
|
||||
将 layers.<num>. 归一化为 layers.*.，用于结构级去重
|
||||
"""
|
||||
return re.sub(r"model.layers\.\d+\.", "model.layers.*.", weight_name)
|
||||
|
||||
|
||||
def deduplicate_weight_map(weight_map: dict):
|
||||
"""
|
||||
对 weight_map 进行结构去重，只保留第一个出现的结构
|
||||
"""
|
||||
seen = set()
|
||||
deduped = OrderedDict()
|
||||
|
||||
for name, file in weight_map.items():
|
||||
key = structure_key(name)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped[name] = file
|
||||
|
||||
return deduped
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from argparse import ArgumentParser
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
parser.add_argument("--input-dir", type=str)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
filter(args.input_dir)
|
||||
|
||||
|
||||
|
||||
189
dev/tools/group_weight_map_structure.py
Normal file
189
dev/tools/group_weight_map_structure.py
Normal file
@@ -0,0 +1,189 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
LAYER_KEY_PATTERN = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
|
||||
CANDIDATE_INDEX_FILES = [
|
||||
"model.safetensors.index.json",
|
||||
"consolidated.safetensors.index.json",
|
||||
]
|
||||
|
||||
|
||||
def normalize_numeric_segments(name: str) -> str:
|
||||
"""
|
||||
Replace pure numeric path segments with '*'.
|
||||
Example: model.layers.12.mlp.experts.3.up_proj.weight
|
||||
-> model.layers.*.mlp.experts.*.up_proj.weight
|
||||
"""
|
||||
return ".".join("*" if part.isdigit() else part for part in name.split("."))
|
||||
|
||||
|
||||
def load_weight_map(input_json_path: str) -> dict[str, str]:
|
||||
with open(input_json_path, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if isinstance(data, dict) and isinstance(data.get("weight_map"), dict):
|
||||
return data["weight_map"]
|
||||
|
||||
if isinstance(data, dict) and all(
|
||||
isinstance(k, str) and isinstance(v, str) for k, v in data.items()
|
||||
):
|
||||
return data
|
||||
|
||||
raise ValueError(
|
||||
"Input json must be either {'weight_map': {...}} or a plain key->file mapping."
|
||||
)
|
||||
|
||||
|
||||
def find_index_json(input_dir: str) -> str:
|
||||
for name in CANDIDATE_INDEX_FILES:
|
||||
path = os.path.join(input_dir, name)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
raise ValueError(f"Can not find index file in {input_dir}")
|
||||
|
||||
|
||||
def load_weight_map_from_input_dir(input_dir: str) -> tuple[dict[str, str], str]:
|
||||
index_json_path = find_index_json(input_dir)
|
||||
return load_weight_map(index_json_path), index_json_path
|
||||
|
||||
|
||||
def build_structure_report(
|
||||
weight_map: dict[str, str],
|
||||
max_examples: int,
|
||||
max_pattern_preview: int,
|
||||
include_all_layer_patterns: bool,
|
||||
) -> dict:
|
||||
pattern_to_keys: dict[str, list[str]] = defaultdict(list)
|
||||
layer_to_suffix_patterns: dict[int, set[str]] = defaultdict(set)
|
||||
non_layer_patterns: set[str] = set()
|
||||
|
||||
for key in weight_map.keys():
|
||||
normalized_full = normalize_numeric_segments(key)
|
||||
pattern_to_keys[normalized_full].append(key)
|
||||
|
||||
layer_match = LAYER_KEY_PATTERN.match(key)
|
||||
if layer_match is None:
|
||||
non_layer_patterns.add(normalized_full)
|
||||
continue
|
||||
|
||||
layer_id = int(layer_match.group(1))
|
||||
layer_suffix = layer_match.group(2)
|
||||
normalized_suffix = normalize_numeric_segments(layer_suffix)
|
||||
layer_to_suffix_patterns[layer_id].add(normalized_suffix)
|
||||
|
||||
normalized_key_patterns = []
|
||||
for pattern, keys in sorted(
|
||||
pattern_to_keys.items(),
|
||||
key=lambda item: (-len(item[1]), item[0]),
|
||||
):
|
||||
normalized_key_patterns.append(
|
||||
{
|
||||
"pattern": pattern,
|
||||
"count": len(keys),
|
||||
"example_keys": sorted(keys)[:max_examples],
|
||||
}
|
||||
)
|
||||
|
||||
all_patterns = [item["pattern"] for item in normalized_key_patterns]
|
||||
|
||||
signature_to_layers: dict[tuple[str, ...], list[int]] = defaultdict(list)
|
||||
for layer_id, suffix_patterns in layer_to_suffix_patterns.items():
|
||||
signature = tuple(sorted(suffix_patterns))
|
||||
signature_to_layers[signature].append(layer_id)
|
||||
|
||||
layer_structure_groups = []
|
||||
sorted_signature_items = sorted(
|
||||
signature_to_layers.items(),
|
||||
key=lambda item: (len(item[1]), item[1][0]),
|
||||
reverse=True,
|
||||
)
|
||||
for idx, (signature, layer_ids) in enumerate(sorted_signature_items, start=1):
|
||||
layer_ids = sorted(layer_ids)
|
||||
entry = {
|
||||
"group_id": f"layer_struct_{idx:03d}",
|
||||
"layer_ids": layer_ids,
|
||||
"layer_count": len(layer_ids),
|
||||
"pattern_count": len(signature),
|
||||
"pattern_preview": list(signature)[:max_pattern_preview],
|
||||
}
|
||||
if include_all_layer_patterns:
|
||||
entry["patterns"] = list(signature)
|
||||
layer_structure_groups.append(entry)
|
||||
|
||||
return {
|
||||
"all_patterns": all_patterns,
|
||||
"total_weight_keys": len(weight_map),
|
||||
"normalized_key_pattern_count": len(normalized_key_patterns),
|
||||
"normalized_key_patterns": normalized_key_patterns,
|
||||
"layer_ids_seen": sorted(layer_to_suffix_patterns.keys()),
|
||||
"layer_structure_group_count": len(layer_structure_groups),
|
||||
"layer_structure_groups": layer_structure_groups,
|
||||
"non_layer_pattern_count": len(non_layer_patterns),
|
||||
"non_layer_patterns": sorted(non_layer_patterns),
|
||||
}
|
||||
|
||||
|
||||
def get_default_output_path(input_json_path: str) -> str:
|
||||
input_path = Path(input_json_path)
|
||||
return str(input_path.with_name(f"{input_path.stem}.layer_structures.json"))
|
||||
|
||||
|
||||
def resolve_io_paths(
|
||||
input_dir: str | None, input_json: str | None, output_json: str | None
|
||||
) -> tuple[dict[str, str], str, str]:
|
||||
if not input_dir and not input_json:
|
||||
raise ValueError("Either --input-dir or --input-json is required.")
|
||||
if input_dir and input_json:
|
||||
raise ValueError("Please provide only one of --input-dir or --input-json.")
|
||||
|
||||
if input_dir:
|
||||
weight_map, input_json_path = load_weight_map_from_input_dir(input_dir)
|
||||
else:
|
||||
input_json_path = input_json
|
||||
weight_map = load_weight_map(input_json_path)
|
||||
|
||||
output_json_path = output_json or get_default_output_path(input_json_path)
|
||||
return weight_map, input_json_path, output_json_path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Group weight_map keys by normalized pattern and layer structure."
|
||||
)
|
||||
parser.add_argument("--input-dir", default=None, type=str)
|
||||
parser.add_argument("--input-json", default=None, type=str)
|
||||
parser.add_argument("--output-json", default=None, type=str)
|
||||
parser.add_argument("--max-examples", default=3, type=int)
|
||||
parser.add_argument("--max-pattern-preview", default=20, type=int)
|
||||
parser.add_argument("--include-all-layer-patterns", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
weight_map, input_json_path, output_json = resolve_io_paths(
|
||||
input_dir=args.input_dir,
|
||||
input_json=args.input_json,
|
||||
output_json=args.output_json,
|
||||
)
|
||||
report = build_structure_report(
|
||||
weight_map=weight_map,
|
||||
max_examples=args.max_examples,
|
||||
max_pattern_preview=args.max_pattern_preview,
|
||||
include_all_layer_patterns=args.include_all_layer_patterns,
|
||||
)
|
||||
|
||||
with open(output_json, "w") as f:
|
||||
json.dump(report, f, indent=2)
|
||||
|
||||
print(f"Saved grouped structure report to: {output_json}")
|
||||
print(f"Loaded weight_map from: {input_json_path}")
|
||||
print(f"Total keys: {report['total_weight_keys']}")
|
||||
print(f"Layer structure groups: {report['layer_structure_group_count']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user