初始化项目，由ModelHub XC社区提供模型
Model: Marco333/qwen2.5-0.5b-game-commands-stt Source: Original Platform
This commit is contained in:
ModelHub XC
358
create_dataset.py
Normal file
358
create_dataset.py
Normal file
@@ -0,0 +1,358 @@
|
||||
"""
|
||||
Generate a synthetic dataset for: Noisy STT text -> Game Command matching.
|
||||
|
||||
This creates training data where:
|
||||
- Input: noisy/garbled speech-to-text transcriptions (as a player would speak into mic)
|
||||
- Output: JSON with the canonical game command
|
||||
|
||||
Based on the approach from arxiv:2502.12923 (synthetic command data + structured JSON output).
|
||||
"""
|
||||
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
# =============================================================================
|
||||
# GAME COMMANDS - Define your game's command vocabulary
|
||||
# =============================================================================
|
||||
GAME_COMMANDS = {
|
||||
"OpenDoor": {
|
||||
"description": "Open a door",
|
||||
"canonical_phrases": [
|
||||
"open the door", "open door", "door open", "open it",
|
||||
"open that door", "can you open the door", "open the gate",
|
||||
"unlock and open", "open sesame", "let me through"
|
||||
]
|
||||
},
|
||||
"CloseDoor": {
|
||||
"description": "Close a door",
|
||||
"canonical_phrases": [
|
||||
"close the door", "close door", "shut the door", "shut it",
|
||||
"close that", "slam the door", "shut that door"
|
||||
]
|
||||
},
|
||||
"Attack": {
|
||||
"description": "Attack an enemy",
|
||||
"canonical_phrases": [
|
||||
"attack", "attack enemy", "hit them", "strike", "fight",
|
||||
"attack now", "kill it", "go attack", "charge", "swing at them",
|
||||
"slash", "hit it", "attack that one"
|
||||
]
|
||||
},
|
||||
"Defend": {
|
||||
"description": "Raise defense/shield",
|
||||
"canonical_phrases": [
|
||||
"defend", "block", "raise shield", "shield up", "guard",
|
||||
"protect me", "defensive stance", "hold block", "shield"
|
||||
]
|
||||
},
|
||||
"Heal": {
|
||||
"description": "Use healing ability or potion",
|
||||
"canonical_phrases": [
|
||||
"heal", "heal me", "use potion", "drink potion", "health potion",
|
||||
"heal up", "restore health", "use heal", "cast heal", "healing"
|
||||
]
|
||||
},
|
||||
"UseItem": {
|
||||
"description": "Use an item from inventory",
|
||||
"canonical_phrases": [
|
||||
"use item", "use it", "activate item", "use the item",
|
||||
"consume item", "use that", "equip and use"
|
||||
]
|
||||
},
|
||||
"PickUp": {
|
||||
"description": "Pick up an item from the ground",
|
||||
"canonical_phrases": [
|
||||
"pick up", "pick it up", "grab it", "take it", "collect",
|
||||
"pick that up", "loot", "grab that", "get it", "take that"
|
||||
]
|
||||
},
|
||||
"DropItem": {
|
||||
"description": "Drop an item",
|
||||
"canonical_phrases": [
|
||||
"drop it", "drop item", "throw away", "discard", "put it down",
|
||||
"drop that", "leave it", "toss it"
|
||||
]
|
||||
},
|
||||
"Jump": {
|
||||
"description": "Make character jump",
|
||||
"canonical_phrases": [
|
||||
"jump", "jump up", "hop", "leap", "jump over", "jump now",
|
||||
"double jump", "jump higher"
|
||||
]
|
||||
},
|
||||
"Crouch": {
|
||||
"description": "Make character crouch/duck",
|
||||
"canonical_phrases": [
|
||||
"crouch", "duck", "crouch down", "get down", "hide",
|
||||
"stay low", "go low", "sneak"
|
||||
]
|
||||
},
|
||||
"Sprint": {
|
||||
"description": "Start sprinting/running fast",
|
||||
"canonical_phrases": [
|
||||
"sprint", "run", "run fast", "go fast", "sprint now",
|
||||
"faster", "hurry", "rush", "quick run", "speed up"
|
||||
]
|
||||
},
|
||||
"Stop": {
|
||||
"description": "Stop moving",
|
||||
"canonical_phrases": [
|
||||
"stop", "halt", "stay", "don't move", "freeze", "hold",
|
||||
"wait", "stop moving", "stand still"
|
||||
]
|
||||
},
|
||||
"Reload": {
|
||||
"description": "Reload weapon",
|
||||
"canonical_phrases": [
|
||||
"reload", "reload weapon", "load ammo", "reload gun",
|
||||
"refill ammo", "reload now", "need ammo"
|
||||
]
|
||||
},
|
||||
"SwitchWeapon": {
|
||||
"description": "Switch to another weapon",
|
||||
"canonical_phrases": [
|
||||
"switch weapon", "change weapon", "next weapon", "swap weapon",
|
||||
"other weapon", "switch gun", "switch to sword", "equip weapon"
|
||||
]
|
||||
},
|
||||
"CastSpell": {
|
||||
"description": "Cast a magic spell",
|
||||
"canonical_phrases": [
|
||||
"cast spell", "use magic", "fire spell", "cast it",
|
||||
"magic attack", "spell now", "use ability", "cast fireball"
|
||||
]
|
||||
},
|
||||
"Interact": {
|
||||
"description": "Interact with an object or NPC",
|
||||
"canonical_phrases": [
|
||||
"interact", "talk to them", "use that", "activate",
|
||||
"press button", "pull lever", "talk", "interact with it"
|
||||
]
|
||||
},
|
||||
"OpenInventory": {
|
||||
"description": "Open the inventory menu",
|
||||
"canonical_phrases": [
|
||||
"open inventory", "inventory", "show inventory", "check inventory",
|
||||
"open bag", "open backpack", "show items", "my items"
|
||||
]
|
||||
},
|
||||
"OpenMap": {
|
||||
"description": "Open the map",
|
||||
"canonical_phrases": [
|
||||
"open map", "show map", "map", "where am I",
|
||||
"check map", "look at map", "navigate"
|
||||
]
|
||||
},
|
||||
"SaveGame": {
|
||||
"description": "Save the game",
|
||||
"canonical_phrases": [
|
||||
"save game", "save", "quick save", "save now",
|
||||
"save progress", "save it"
|
||||
]
|
||||
},
|
||||
"PauseGame": {
|
||||
"description": "Pause the game",
|
||||
"canonical_phrases": [
|
||||
"pause", "pause game", "pause it", "hold on",
|
||||
"time out", "break", "stop game"
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# ASR NOISE SIMULATION
|
||||
# =============================================================================
|
||||
|
||||
PHONETIC_NOISE = {
|
||||
"open": ["opun", "opin", "o pen", "owen", "oben", "oven", "oppn"],
|
||||
"close": ["cloze", "cloes", "klose", "cloz", "clothe", "clos"],
|
||||
"door": ["dur", "dor", "doer", "dore", "dour", "tor"],
|
||||
"attack": ["attak", "a tack", "attac", "atak", "attic", "a tech"],
|
||||
"enemy": ["enemi", "enamy", "enemy", "emeny", "ememy"],
|
||||
"defend": ["defnd", "defent", "difend", "de fend", "def end"],
|
||||
"shield": ["sheeld", "shild", "sheld", "sheild", "chield"],
|
||||
"heal": ["hel", "heel", "he'll", "heell", "hil", "hale"],
|
||||
"potion": ["posion", "poshan", "potian", "potion", "poshin"],
|
||||
"pick": ["pic", "pik", "peek", "prick", "pig"],
|
||||
"jump": ["jum", "jamp", "jomp", "jup", "gump", "dump"],
|
||||
"crouch": ["crowch", "krauch", "croush", "krouch", "couch"],
|
||||
"sprint": ["spint", "sprnt", "sprent", "srint", "print"],
|
||||
"reload": ["relod", "re load", "ree load", "relode", "reelod"],
|
||||
"weapon": ["wepon", "weppon", "wapon", "weapn", "weapen"],
|
||||
"switch": ["swich", "swish", "swithc", "svitch", "stitch"],
|
||||
"spell": ["spel", "spal", "spall", "spell", "speel"],
|
||||
"cast": ["kast", "cas", "cass", "karst", "cast"],
|
||||
"inventory": ["inventery", "inventry", "inventor", "invetory", "inventori"],
|
||||
"map": ["mop", "nap", "map", "maps", "mab"],
|
||||
"save": ["sav", "safe", "saev", "sayve", "saf"],
|
||||
"pause": ["paws", "pouse", "paus", "pawse", "pos"],
|
||||
"item": ["ithem", "iten", "I tem", "itam", "aitem"],
|
||||
"grab": ["grap", "grabe", "gab", "garb", "crab"],
|
||||
"stop": ["stob", "stap", "stopp", "sop", "top"],
|
||||
"run": ["rn", "runn", "wrun", "ran", "rune"],
|
||||
"fast": ["fass", "fas", "farst", "fust", "fest"],
|
||||
"use": ["yuse", "uz", "ewse", "uce", "oose"],
|
||||
"fire": ["fir", "fyre", "fiar", "fier", "fre"],
|
||||
"magic": ["majic", "magik", "madgic", "majik", "magc"],
|
||||
"game": ["gaem", "gam", "gaim", "gme", "dame"],
|
||||
"talk": ["tok", "tawk", "tolk", "tak", "talc"],
|
||||
"button": ["buton", "buttn", "baton", "botton", "buttan"],
|
||||
"lever": ["leaver", "lever", "leva", "levr", "liver"],
|
||||
"up": ["op", "uhp", "ap", "ub"],
|
||||
"down": ["don", "doun", "daun", "dawn"],
|
||||
"the": ["da", "de", "teh", "duh", "tha", ""],
|
||||
"it": ["et", "ih", "at", "i", "id"],
|
||||
"that": ["dat", "tat", "thet", "thad"],
|
||||
"now": ["naw", "nou", "naow", "no"],
|
||||
}
|
||||
|
||||
BG_NOISE_INSERTIONS = ["um", "uh", "hmm", "ah", "like", "you know", "err", "", "", ""]
|
||||
|
||||
|
||||
def apply_formatting_noise(text):
|
||||
noise_type = random.random()
|
||||
if noise_type < 0.15:
|
||||
text = text.lower()
|
||||
elif noise_type < 0.25:
|
||||
text = ''.join(c.upper() if random.random() < 0.2 else c.lower() for c in text)
|
||||
elif noise_type < 0.35:
|
||||
words = text.split()
|
||||
if len(words) > 1:
|
||||
idx = random.randint(0, len(words)-2)
|
||||
words[idx] = words[idx] + words[idx+1]
|
||||
del words[idx+1]
|
||||
text = ' '.join(words)
|
||||
elif noise_type < 0.4:
|
||||
text = re.sub(r' ', ' ', text, count=1)
|
||||
return text
|
||||
|
||||
|
||||
def inject_asr_noise(text, noise_level=0.4):
|
||||
words = text.lower().split()
|
||||
result = []
|
||||
for i, word in enumerate(words):
|
||||
if random.random() < 0.05 * noise_level:
|
||||
continue
|
||||
if word in PHONETIC_NOISE and random.random() < noise_level:
|
||||
result.append(random.choice(PHONETIC_NOISE[word]))
|
||||
elif random.random() < noise_level * 0.3 and len(word) > 3:
|
||||
noise_ops = ['swap', 'delete', 'insert', 'replace']
|
||||
op = random.choice(noise_ops)
|
||||
w = list(word)
|
||||
if op == 'swap' and len(w) > 2:
|
||||
idx = random.randint(0, len(w)-2)
|
||||
w[idx], w[idx+1] = w[idx+1], w[idx]
|
||||
elif op == 'delete' and len(w) > 2:
|
||||
idx = random.randint(0, len(w)-1)
|
||||
del w[idx]
|
||||
elif op == 'insert':
|
||||
idx = random.randint(0, len(w))
|
||||
w.insert(idx, random.choice('abcdefghijklmnopqrstuvwxyz'))
|
||||
elif op == 'replace':
|
||||
idx = random.randint(0, len(w)-1)
|
||||
w[idx] = random.choice('abcdefghijklmnopqrstuvwxyz')
|
||||
result.append(''.join(w))
|
||||
else:
|
||||
result.append(word)
|
||||
if random.random() < 0.08 * noise_level:
|
||||
filler = random.choice(BG_NOISE_INSERTIONS)
|
||||
if filler:
|
||||
result.append(filler)
|
||||
if random.random() < 0.1 * noise_level and len(result) > 0:
|
||||
idx = random.randint(0, len(result)-1)
|
||||
result.insert(idx, result[idx])
|
||||
text = ' '.join(result)
|
||||
text = apply_formatting_noise(text)
|
||||
return text
|
||||
|
||||
|
||||
def generate_variations(phrase, num_variations=5):
|
||||
variations = []
|
||||
for _ in range(num_variations):
|
||||
noise_level = random.uniform(0.2, 0.8)
|
||||
noisy = inject_asr_noise(phrase, noise_level)
|
||||
if noisy.strip() and noisy != phrase.lower():
|
||||
variations.append(noisy)
|
||||
variations.append(phrase.lower())
|
||||
variations.append(phrase)
|
||||
return variations
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """You are a game command interpreter. The user's input is noisy speech-to-text from a microphone in a gaming environment. Your job is to identify the intended game command from the noisy input.
|
||||
|
||||
Valid commands: {commands}
|
||||
|
||||
Respond with ONLY a JSON object: {{"command": "<CommandName>"}}
|
||||
If the input doesn't match any command, respond: {{"command": "Unknown"}}"""
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
command_list = ", ".join(sorted(GAME_COMMANDS.keys()))
|
||||
return SYSTEM_PROMPT.format(commands=command_list)
|
||||
|
||||
|
||||
def generate_dataset(num_samples_per_command=100, include_negatives=True):
|
||||
samples = []
|
||||
system_msg = get_system_prompt()
|
||||
for cmd_name, cmd_info in GAME_COMMANDS.items():
|
||||
for phrase in cmd_info["canonical_phrases"]:
|
||||
num_vars = max(3, num_samples_per_command // len(cmd_info["canonical_phrases"]))
|
||||
variations = generate_variations(phrase, num_variations=num_vars)
|
||||
for noisy_text in variations:
|
||||
if not noisy_text.strip():
|
||||
continue
|
||||
sample = {
|
||||
"messages": [
|
||||
{"role": "system", "content": system_msg},
|
||||
{"role": "user", "content": noisy_text},
|
||||
{"role": "assistant", "content": json.dumps({"command": cmd_name})}
|
||||
]
|
||||
}
|
||||
samples.append(sample)
|
||||
if include_negatives:
|
||||
negative_phrases = [
|
||||
"what time is it", "hello how are you", "this game is awesome",
|
||||
"I need to go to the bathroom", "what's the score", "can you hear me",
|
||||
"testing testing one two three", "hold on let me think", "that was close",
|
||||
"nice shot dude", "where are we going", "I don't know what to do",
|
||||
"this is so hard", "let's take a different route", "watch out behind you",
|
||||
"I think we should go left", "what level are you",
|
||||
"how much health do you have", "we need more players",
|
||||
"good game everyone", "random noise blah blah", "shshhshshs",
|
||||
"mmmhmm yeah", "no no no wait", "okay okay I got it",
|
||||
]
|
||||
for phrase in negative_phrases:
|
||||
for _ in range(8):
|
||||
noisy = inject_asr_noise(phrase, noise_level=random.uniform(0.1, 0.5))
|
||||
sample = {
|
||||
"messages": [
|
||||
{"role": "system", "content": system_msg},
|
||||
{"role": "user", "content": noisy},
|
||||
{"role": "assistant", "content": json.dumps({"command": "Unknown"})}
|
||||
]
|
||||
}
|
||||
samples.append(sample)
|
||||
random.shuffle(samples)
|
||||
return samples
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
random.seed(42)
|
||||
print("Generating training dataset...")
|
||||
train_samples = generate_dataset(num_samples_per_command=120)
|
||||
print(f" Training samples: {len(train_samples)}")
|
||||
|
||||
random.seed(123)
|
||||
print("Generating test dataset...")
|
||||
test_samples = generate_dataset(num_samples_per_command=20)
|
||||
print(f" Test samples: {len(test_samples)}")
|
||||
|
||||
train_ds = Dataset.from_list(train_samples)
|
||||
test_ds = Dataset.from_list(test_samples)
|
||||
dataset_dict = DatasetDict({"train": train_ds, "test": test_ds})
|
||||
|
||||
print(f"\nPushing to Hub...")
|
||||
dataset_dict.push_to_hub("Marco333/game-commands-noisy-stt", private=False)
|
||||
print("Done!")
|
||||
Reference in New Issue
Block a user