初始化项目,由ModelHub XC社区提供模型
Model: voidful/llm-codec Source: Original Platform
This commit is contained in:
commit
66598540f8
13 files changed
+546
No files matched your search
@@ -0,0 +1,59 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
llm-codec.pt filter=lfs diff=lfs merge=lfs -text
|
||||
model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
vocab.json filter=lfs diff=lfs merge=lfs -text
|
||||
merges.txt filter=lfs diff=lfs merge=lfs -text
|
||||
model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -0,0 +1,292 @@
|
||||
---
|
||||
base_model:
|
||||
- Qwen/Qwen3-4B-Instruct-2507
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
tags:
|
||||
- audio
|
||||
- speech
|
||||
- audio-codec
|
||||
- neural-audio-codec
|
||||
- spoken-language-modeling
|
||||
- codec-superb
|
||||
- qwen3
|
||||
datasets:
|
||||
- librispeech_asr
|
||||
metrics:
|
||||
- perplexity
|
||||
- pesq
|
||||
- stoi
|
||||
---
|
||||
|
||||
# LLM-Codec
|
||||
|
||||
LLM-Codec is a neural audio codec checkpoint trained to produce discrete audio
|
||||
tokens that are both reconstructable and easier for autoregressive language
|
||||
models to predict.
|
||||
|
||||
Model: https://huggingface.co/voidful/llm-codec
|
||||
|
||||
Code: https://github.com/voidful/llm-codec
|
||||
|
||||
Usage reference: https://github.com/voidful/Codec-SUPERB
|
||||
|
||||
## Model Description
|
||||
|
||||
Most neural audio codecs are trained for waveform reconstruction. Spoken
|
||||
language models, however, consume codec tokens with a next-token prediction
|
||||
objective. This mismatch can make acoustically valid variation appear as token
|
||||
uncertainty to the language model.
|
||||
|
||||
LLM-Codec adapts a codec with language-model-facing objectives while keeping the
|
||||
deployed codec interface unchanged. The model is trained with:
|
||||
|
||||
- Future Token Prediction (FTP): Medusa-style heads predict future audio tokens
|
||||
from frozen-LLM hidden states.
|
||||
- Semantic Alignment (SA): audio-induced hidden states are aligned with paired
|
||||
text hidden states inside a frozen LLM.
|
||||
- Differentiable Gumbel bridge: hard Gumbel-Softmax keeps discrete forward
|
||||
tokens while enabling gradients to flow to the codec encoder.
|
||||
- Reconstruction losses: mel, multi-scale mel, multi-resolution STFT, complex
|
||||
STFT, VQ, GAN, and feature matching losses.
|
||||
|
||||
The deployed codec does not require the auxiliary FTP heads.
|
||||
|
||||
## Intended Use
|
||||
|
||||
This model is intended for research and development in:
|
||||
|
||||
- audio tokenization for spoken language modeling
|
||||
- codec reconstruction experiments
|
||||
- token-level speech LM training
|
||||
- Codec-SUPERB style codec evaluation
|
||||
- speech token analysis and ablation studies
|
||||
|
||||
It is not a full text-to-speech system by itself. For speech generation, use the
|
||||
codec as the tokenizer/decoder inside a separate speech language modeling
|
||||
pipeline.
|
||||
|
||||
## Out-of-Scope Use
|
||||
|
||||
Do not use this model for:
|
||||
|
||||
- impersonation or unauthorized voice cloning
|
||||
- surveillance or speaker tracking without consent
|
||||
- high-stakes speaker, language, or identity decisions
|
||||
- generating deceptive audio content
|
||||
|
||||
## Installation
|
||||
|
||||
The easiest inference path is through the Codec-SUPERB `SoundCodec` interface.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/voidful/Codec-SUPERB.git
|
||||
cd Codec-SUPERB
|
||||
pip install -r requirements.txt
|
||||
export PYTHONPATH=$PWD:$PYTHONPATH
|
||||
```
|
||||
|
||||
If your environment supports editable installs, this is also convenient:
|
||||
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Load LLM-Codec through the Codec-SUPERB codec registry:
|
||||
|
||||
```python
|
||||
from SoundCodec import codec
|
||||
|
||||
print(codec.list_codec())
|
||||
model = codec.load_codec("llmcodec")
|
||||
```
|
||||
|
||||
Encode and reconstruct one audio file:
|
||||
|
||||
```python
|
||||
from SoundCodec import codec
|
||||
import torchaudio
|
||||
import soundfile as sf
|
||||
|
||||
model = codec.load_codec("llmcodec")
|
||||
|
||||
waveform, sample_rate = torchaudio.load("sample_audio.wav")
|
||||
data_item = {
|
||||
"audio": {
|
||||
"array": waveform.numpy()[0],
|
||||
"sampling_rate": sample_rate,
|
||||
}
|
||||
}
|
||||
|
||||
units = model.extract_unit(data_item).unit
|
||||
print("Unit shape:", units.shape)
|
||||
|
||||
result = model.synth(data_item, local_save=False)
|
||||
reconstructed = result["audio"]["array"]
|
||||
reconstructed_sr = result["audio"].get("sampling_rate", sample_rate)
|
||||
|
||||
sf.write("reconstructed.wav", reconstructed, reconstructed_sr)
|
||||
```
|
||||
|
||||
## Batch Usage
|
||||
|
||||
Codec-SUPERB also provides batch APIs:
|
||||
|
||||
```python
|
||||
from SoundCodec import codec
|
||||
import torchaudio
|
||||
|
||||
model = codec.load_codec("llmcodec")
|
||||
|
||||
audio_files = ["audio1.wav", "audio2.wav", "audio3.wav"]
|
||||
data_list = []
|
||||
|
||||
for path in audio_files:
|
||||
waveform, sample_rate = torchaudio.load(path)
|
||||
data_list.append({
|
||||
"id": path,
|
||||
"audio": {
|
||||
"array": waveform.numpy()[0],
|
||||
"sampling_rate": sample_rate,
|
||||
},
|
||||
})
|
||||
|
||||
batch_units = model.batch_extract_unit(data_list)
|
||||
batch_audio = model.batch_decode_unit(batch_units)
|
||||
|
||||
results = model.batch_synth(data_list, local_save=False)
|
||||
for item in results:
|
||||
print(item["unit"].shape, item["audio"]["array"].shape)
|
||||
```
|
||||
|
||||
For better throughput, group audio samples with similar lengths before batching.
|
||||
|
||||
## Codec-SUPERB Evaluation
|
||||
|
||||
To evaluate LLM-Codec with Codec-SUPERB-tiny:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=. python3 scripts/dataset_creator.py \
|
||||
--dataset voidful/codec-superb-tiny
|
||||
|
||||
PYTHONPATH=. python3 scripts/benchmarking.py \
|
||||
--dataset datasets/voidful/codec-superb-tiny_synth \
|
||||
--models llmcodec
|
||||
```
|
||||
|
||||
## Model Files
|
||||
|
||||
The model repository provides:
|
||||
|
||||
- codec weights as `llm-codec.pt`
|
||||
- a tokenizer extended with `<CODEC_*>` audio tokens
|
||||
- Qwen-compatible model artifacts containing trained audio-token embeddings
|
||||
|
||||
The codec uses 20,480 audio tokens with the canonical token format:
|
||||
|
||||
```text
|
||||
<CODEC_0>, <CODEC_1>, ..., <CODEC_20479>
|
||||
```
|
||||
|
||||
## Training Data
|
||||
|
||||
The codec was trained on LibriSpeech `train-clean-100` with paired transcripts.
|
||||
The validation split used during training is LibriSpeech `validation`.
|
||||
|
||||
Because training is speech-centric and transcript-supervised, performance may be
|
||||
weaker on non-English speech, conversational speech, music, environmental audio,
|
||||
or audio with strong noise and overlap.
|
||||
|
||||
## Training Procedure
|
||||
|
||||
Base components:
|
||||
|
||||
- Base codec: AUV
|
||||
- Frozen LLM backbone: Qwen3-4B-Instruct
|
||||
- Token rate: 50 Hz
|
||||
- Audio vocabulary size: 20,480
|
||||
- Segment length: 4 seconds
|
||||
|
||||
Losses:
|
||||
|
||||
- reconstruction mel loss
|
||||
- multi-scale mel loss
|
||||
- multi-resolution STFT loss
|
||||
- complex STFT loss with phase term
|
||||
- VQ commitment loss
|
||||
- Gumbel bridge cross entropy
|
||||
- Future Token Prediction loss
|
||||
- Semantic Alignment cosine loss
|
||||
- Semantic Alignment contrastive loss with memory bank
|
||||
- MPD/MSD GAN and feature matching losses
|
||||
|
||||
## Evaluation Results
|
||||
|
||||
### Token Learnability
|
||||
|
||||
SALMon speech coherence accuracy after token-level LM training:
|
||||
|
||||
| Tokenizer | Overall accuracy |
|
||||
| --- | ---: |
|
||||
| WavTok-L | 48.3 |
|
||||
| BigCodec | 49.4 |
|
||||
| UniCodec | 50.1 |
|
||||
| AUV | 49.4 |
|
||||
| LLM-Codec | 61.6 |
|
||||
|
||||
Token-level perplexity on LibriSpeech after 3 epochs of LM training:
|
||||
|
||||
| Tokenizer | Eval loss | Perplexity |
|
||||
| --- | ---: | ---: |
|
||||
| WavTok-L | 11.91 | 148,122 |
|
||||
| UniCodec | 11.92 | 150,197 |
|
||||
| BigCodec | 11.96 | 156,448 |
|
||||
| AUV | 11.98 | 159,768 |
|
||||
| LLM-Codec | 8.44 | 4,617 |
|
||||
|
||||
### Reconstruction Quality
|
||||
|
||||
Codec-SUPERB-tiny speech reconstruction:
|
||||
|
||||
| Model | Mel lower is better | STFT lower is better | PESQ higher is better | STOI higher is better |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| AUV base | 0.762 | 1.648 | 2.094 | 0.850 |
|
||||
| LLM-Codec | 0.724 | 1.599 | 2.102 | 0.859 |
|
||||
|
||||
## Limitations
|
||||
|
||||
- The semantic alignment objective depends on paired speech and text.
|
||||
- The model is primarily validated on read speech.
|
||||
- Downstream generation quality depends on the separate speech language model.
|
||||
- The model may preserve speaker identity information present in the input.
|
||||
- The Hugging Face `transformers` artifacts are not a standalone text chatbot;
|
||||
they accompany the codec/tokenizer workflow.
|
||||
|
||||
## Citation
|
||||
|
||||
```bibtex
|
||||
@article{chung2026llm,
|
||||
title={LLM-Codec: Neural Audio Codec Meets Language Model Objectives},
|
||||
author={Chung, Ho-Lam and Chen, Yiming and Lee, Hung-yi},
|
||||
journal={arXiv preprint arXiv:2604.17852},
|
||||
note = {Model and code available at https://github.com/voidful/llm-codec},
|
||||
year={2026}
|
||||
}
|
||||
```
|
||||
|
||||
If you use the Codec-SUPERB interface or benchmark, please also cite
|
||||
Codec-SUPERB:
|
||||
|
||||
```bibtex
|
||||
@inproceedings{wu-etal-2024-codec,
|
||||
title = {Codec-SUPERB: An In-Depth Analysis of Sound Codec Models},
|
||||
author = {Wu, Haibin and Chung, Ho-Lam and Lin, Yi-Cheng and Wu, Yuan-Kuei and Chen, Xuanjun and Pai, Yu-Chi and Wang, Hsiu-Hsuan and Chang, Kai-Wei and Liu, Alexander and Lee, Hung-yi},
|
||||
booktitle = {Findings of the Association for Computational Linguistics: ACL 2024},
|
||||
year = {2024},
|
||||
url = {https://aclanthology.org/2024.findings-acl.616},
|
||||
doi = {10.18653/v1/2024.findings-acl.616},
|
||||
pages = {10330--10348}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,61 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- messages[0].content + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if message.content is string %}
|
||||
{%- set content = message.content %}
|
||||
{%- else %}
|
||||
{%- set content = '' %}
|
||||
{%- endif %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2560,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 9728,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 262144,
|
||||
"max_window_layers": 36,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": null,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 5000000,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.4.0",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 172149
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.4.0"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:76067770b0939d17b1560532edce18de75953db6db5eaf063803a1b713ea1f48
|
||||
size 490842187
|
||||
LFS
BIN
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:83e300b1d75f0a2dca1f57fbb22be6b8d28d7ecab64804999f85e8872b319326
|
||||
size 8148472656
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:57b368b5d5fa28d222078ba59cea9a189d6dfdb0bb8fde811819a65207f45056
|
||||
size 15302740
|
||||
BIN
Binary file not shown.
LFS
BIN
Binary file not shown.
Reference in new issue
Block a user