commit 67224edb23f31754ac96fee0917565508750401f Author: ModelHub XC Date: Thu Jul 9 22:53:11 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: ree2raz/CyberSecQwen-4B-AWQ Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..10fbf2b --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +--- +license: apache-2.0 +base_model: lablab-ai-amd-developer-hackathon/CyberSecQwen-4B +tags: +- qwen3 +- cybersecurity +- cti +- cwe-classification +- vulnerability-analysis +- awq +- 4-bit +- quantized +library_name: transformers +pipeline_tag: text-generation +--- + +# CyberSecQwen-4B-AWQ + +4-bit AWQ quantized version of [CyberSecQwen-4B](https://huggingface.co/lablab-ai-amd-developer-hackathon/CyberSecQwen-4B). + +## Quantization + +| Parameter | Value | +|---|---| +| Method | AWQ (group_size=128, zero_point=True) | +| Weight precision | 4-bit | +| Compute dtype | float16 | +| Calibration samples | 320 CTI-Bench prompts (256 RCM + 64 MCQ, chat-template formatted) | +| Quantization tool | autoawq | +| Calibration hardware | Modal A100 | + +## CTI-Bench Evaluation + +Evaluated under the [Foundation-Sec-8B protocol](https://arxiv.org/abs/2504.21039): +- Temperature 0.3, max_tokens 512, concurrency 32 +- 5 independent trials, zero-shot (no system prompt) +- vLLM v0.20.1 with awq_marlin kernel on Modal L4 GPU + +| Task | AWQ 4-bit | GGUF Q4_K_M | FP16 Reference | +|---|---|---|---| +| CTI-MCQ (2,500 items) | **0.5921** ± 0.0083 | 0.5368 ± 0.0048 | 0.5868 ± 0.0029 | +| CTI-RCM (1,000 items) | 0.5814 ± 0.0025 | **0.6254 ± 0.0063** | 0.6664 ± 0.0023 | + +**Key findings:** +- **CTI-MCQ**: AWQ 4-bit matches or slightly exceeds FP16 performance (+0.5 points). Better than GGUF Q4_K_M. +- **CTI-RCM**: AWQ 4-bit degrades by 8.5 percentage points vs FP16. GGUF Q4_K_M does better on this task (-4.1 pts). +- AWQ is best for MCQ (general language), GGUF is best for RCM (task-specific classification). + +## Trial results + +### CTI-MCQ +| Trial | Seed | Accuracy | +|---|---|---| +| 1 | 42 | 0.6016 | +| 2 | 43 | 0.5984 | +| 3 | 44 | 0.5936 | +| 4 | 45 | 0.5780 | +| 5 | 46 | 0.5888 | + +### CTI-MCQ + +| Trial | Seed | Accuracy | +|---|---|---| +| 1 | 42 | 0.6016 | +| 2 | 43 | 0.5984 | +| 3 | 44 | 0.5936 | +| 4 | 45 | 0.5780 | +| 5 | 46 | 0.5888 | + +### CTI-RCM + +| Trial | Seed | Accuracy | +|---|---|---| +| 1 | 42 | 0.5790 | +| 2 | 43 | 0.5830 | +| 3 | 44 | 0.5790 | +| 4 | 45 | 0.5840 | +| 5 | 46 | 0.5820 | + +## Quantization variants + +| Variant | CTI-MCQ | CTI-RCM | Size | Engine | +|---|---|---|---|---| +| [AWQ 4-bit](https://huggingface.co/ree2raz/CyberSecQwen-4B-AWQ) | 0.5921 | 0.5814 | 2.7 GB | vLLM | +| [GGUF Q4_K_M](https://huggingface.co/ree2raz/CyberSecQwen-4B-GGUF) | 0.5368 | 0.6254 | 2.5 GB | llama.cpp | + +Choose AWQ for MCQ/general chat, GGUF for vulnerability classification. + +## Usage with vLLM + +```bash +vllm serve ree2raz/CyberSecQwen-4B-AWQ --quantization awq_marlin --dtype float16 +``` + +## Model Size + +| Format | Size | +|---|---| +| Original FP16 | ~8 GB | +| AWQ 4-bit | ~2.7 GB | + +## Citation + +```bibtex +@misc{{cybersecqwen2026, + title = {{CyberSecQwen-4B: A Compact CTI Specialist Fine-Tuned from Qwen3-4B-Instruct-2507 on AMD MI300X}}, + author = {{Mulia, Samuel}}, + year = {{2026}}, + publisher = {{Hugging Face}}, + url = {{https://huggingface.co/athena129/CyberSecQwen-4B}} +}} +``` + +## Evaluation Infrastructure + +[GitHub repository](https://github.com/ree2raz/cyberSecQwen_4b_4bit) — Modal scripts for quantization + evaluation. diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..90b2aee --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,7 @@ +{%- for message in messages -%} +<|im_start|>{{ message['role'] if message['role'] != 'system' else 'user' }} +{{ message['content'] }}<|im_end|> +{% endfor -%} +{%- if add_generation_prompt -%} +<|im_start|>assistant +{% endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000..023571b --- /dev/null +++ b/config.json @@ -0,0 +1,79 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "float16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": null, + "quantization_config": { + "bits": 4, + "group_size": 128, + "modules_to_not_convert": null, + "quant_method": "awq", + "version": "gemm", + "zero_point": true + }, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 5000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.8.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..2d8b869 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.8.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..4fabfa4 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:569a1a7abff2dfd25de06769674a6e7ad4f4c8b8267f75bef52a89cb2337a3c2 +size 2666027528 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..cb87962 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}