359 lines
14 KiB
Python
359 lines
14 KiB
Python
|
"""
|
||
|
|
Generate a synthetic dataset for: Noisy STT text -> Game Command matching.
|
||
|
|
|
||
|
|
This creates training data where:
|
||
|
|
- Input: noisy/garbled speech-to-text transcriptions (as a player would speak into mic)
|
||
|
|
- Output: JSON with the canonical game command
|
||
|
|
|
||
|
|
Based on the approach from arxiv:2502.12923 (synthetic command data + structured JSON output).
|
||
|
|
"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import random
|
||
|
|
import re
|
||
|
|
from datasets import Dataset, DatasetDict
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# GAME COMMANDS - Define your game's command vocabulary
|
||
|
|
# =============================================================================
|
||
|
|
GAME_COMMANDS = {
|
||
|
|
"OpenDoor": {
|
||
|
|
"description": "Open a door",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"open the door", "open door", "door open", "open it",
|
||
|
|
"open that door", "can you open the door", "open the gate",
|
||
|
|
"unlock and open", "open sesame", "let me through"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"CloseDoor": {
|
||
|
|
"description": "Close a door",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"close the door", "close door", "shut the door", "shut it",
|
||
|
|
"close that", "slam the door", "shut that door"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Attack": {
|
||
|
|
"description": "Attack an enemy",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"attack", "attack enemy", "hit them", "strike", "fight",
|
||
|
|
"attack now", "kill it", "go attack", "charge", "swing at them",
|
||
|
|
"slash", "hit it", "attack that one"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Defend": {
|
||
|
|
"description": "Raise defense/shield",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"defend", "block", "raise shield", "shield up", "guard",
|
||
|
|
"protect me", "defensive stance", "hold block", "shield"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Heal": {
|
||
|
|
"description": "Use healing ability or potion",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"heal", "heal me", "use potion", "drink potion", "health potion",
|
||
|
|
"heal up", "restore health", "use heal", "cast heal", "healing"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"UseItem": {
|
||
|
|
"description": "Use an item from inventory",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"use item", "use it", "activate item", "use the item",
|
||
|
|
"consume item", "use that", "equip and use"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"PickUp": {
|
||
|
|
"description": "Pick up an item from the ground",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"pick up", "pick it up", "grab it", "take it", "collect",
|
||
|
|
"pick that up", "loot", "grab that", "get it", "take that"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"DropItem": {
|
||
|
|
"description": "Drop an item",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"drop it", "drop item", "throw away", "discard", "put it down",
|
||
|
|
"drop that", "leave it", "toss it"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Jump": {
|
||
|
|
"description": "Make character jump",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"jump", "jump up", "hop", "leap", "jump over", "jump now",
|
||
|
|
"double jump", "jump higher"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Crouch": {
|
||
|
|
"description": "Make character crouch/duck",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"crouch", "duck", "crouch down", "get down", "hide",
|
||
|
|
"stay low", "go low", "sneak"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Sprint": {
|
||
|
|
"description": "Start sprinting/running fast",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"sprint", "run", "run fast", "go fast", "sprint now",
|
||
|
|
"faster", "hurry", "rush", "quick run", "speed up"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Stop": {
|
||
|
|
"description": "Stop moving",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"stop", "halt", "stay", "don't move", "freeze", "hold",
|
||
|
|
"wait", "stop moving", "stand still"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Reload": {
|
||
|
|
"description": "Reload weapon",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"reload", "reload weapon", "load ammo", "reload gun",
|
||
|
|
"refill ammo", "reload now", "need ammo"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"SwitchWeapon": {
|
||
|
|
"description": "Switch to another weapon",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"switch weapon", "change weapon", "next weapon", "swap weapon",
|
||
|
|
"other weapon", "switch gun", "switch to sword", "equip weapon"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"CastSpell": {
|
||
|
|
"description": "Cast a magic spell",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"cast spell", "use magic", "fire spell", "cast it",
|
||
|
|
"magic attack", "spell now", "use ability", "cast fireball"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"Interact": {
|
||
|
|
"description": "Interact with an object or NPC",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"interact", "talk to them", "use that", "activate",
|
||
|
|
"press button", "pull lever", "talk", "interact with it"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"OpenInventory": {
|
||
|
|
"description": "Open the inventory menu",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"open inventory", "inventory", "show inventory", "check inventory",
|
||
|
|
"open bag", "open backpack", "show items", "my items"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"OpenMap": {
|
||
|
|
"description": "Open the map",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"open map", "show map", "map", "where am I",
|
||
|
|
"check map", "look at map", "navigate"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"SaveGame": {
|
||
|
|
"description": "Save the game",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"save game", "save", "quick save", "save now",
|
||
|
|
"save progress", "save it"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"PauseGame": {
|
||
|
|
"description": "Pause the game",
|
||
|
|
"canonical_phrases": [
|
||
|
|
"pause", "pause game", "pause it", "hold on",
|
||
|
|
"time out", "break", "stop game"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# ASR NOISE SIMULATION
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
PHONETIC_NOISE = {
|
||
|
|
"open": ["opun", "opin", "o pen", "owen", "oben", "oven", "oppn"],
|
||
|
|
"close": ["cloze", "cloes", "klose", "cloz", "clothe", "clos"],
|
||
|
|
"door": ["dur", "dor", "doer", "dore", "dour", "tor"],
|
||
|
|
"attack": ["attak", "a tack", "attac", "atak", "attic", "a tech"],
|
||
|
|
"enemy": ["enemi", "enamy", "enemy", "emeny", "ememy"],
|
||
|
|
"defend": ["defnd", "defent", "difend", "de fend", "def end"],
|
||
|
|
"shield": ["sheeld", "shild", "sheld", "sheild", "chield"],
|
||
|
|
"heal": ["hel", "heel", "he'll", "heell", "hil", "hale"],
|
||
|
|
"potion": ["posion", "poshan", "potian", "potion", "poshin"],
|
||
|
|
"pick": ["pic", "pik", "peek", "prick", "pig"],
|
||
|
|
"jump": ["jum", "jamp", "jomp", "jup", "gump", "dump"],
|
||
|
|
"crouch": ["crowch", "krauch", "croush", "krouch", "couch"],
|
||
|
|
"sprint": ["spint", "sprnt", "sprent", "srint", "print"],
|
||
|
|
"reload": ["relod", "re load", "ree load", "relode", "reelod"],
|
||
|
|
"weapon": ["wepon", "weppon", "wapon", "weapn", "weapen"],
|
||
|
|
"switch": ["swich", "swish", "swithc", "svitch", "stitch"],
|
||
|
|
"spell": ["spel", "spal", "spall", "spell", "speel"],
|
||
|
|
"cast": ["kast", "cas", "cass", "karst", "cast"],
|
||
|
|
"inventory": ["inventery", "inventry", "inventor", "invetory", "inventori"],
|
||
|
|
"map": ["mop", "nap", "map", "maps", "mab"],
|
||
|
|
"save": ["sav", "safe", "saev", "sayve", "saf"],
|
||
|
|
"pause": ["paws", "pouse", "paus", "pawse", "pos"],
|
||
|
|
"item": ["ithem", "iten", "I tem", "itam", "aitem"],
|
||
|
|
"grab": ["grap", "grabe", "gab", "garb", "crab"],
|
||
|
|
"stop": ["stob", "stap", "stopp", "sop", "top"],
|
||
|
|
"run": ["rn", "runn", "wrun", "ran", "rune"],
|
||
|
|
"fast": ["fass", "fas", "farst", "fust", "fest"],
|
||
|
|
"use": ["yuse", "uz", "ewse", "uce", "oose"],
|
||
|
|
"fire": ["fir", "fyre", "fiar", "fier", "fre"],
|
||
|
|
"magic": ["majic", "magik", "madgic", "majik", "magc"],
|
||
|
|
"game": ["gaem", "gam", "gaim", "gme", "dame"],
|
||
|
|
"talk": ["tok", "tawk", "tolk", "tak", "talc"],
|
||
|
|
"button": ["buton", "buttn", "baton", "botton", "buttan"],
|
||
|
|
"lever": ["leaver", "lever", "leva", "levr", "liver"],
|
||
|
|
"up": ["op", "uhp", "ap", "ub"],
|
||
|
|
"down": ["don", "doun", "daun", "dawn"],
|
||
|
|
"the": ["da", "de", "teh", "duh", "tha", ""],
|
||
|
|
"it": ["et", "ih", "at", "i", "id"],
|
||
|
|
"that": ["dat", "tat", "thet", "thad"],
|
||
|
|
"now": ["naw", "nou", "naow", "no"],
|
||
|
|
}
|
||
|
|
|
||
|
|
BG_NOISE_INSERTIONS = ["um", "uh", "hmm", "ah", "like", "you know", "err", "", "", ""]
|
||
|
|
|
||
|
|
|
||
|
|
def apply_formatting_noise(text):
|
||
|
|
noise_type = random.random()
|
||
|
|
if noise_type < 0.15:
|
||
|
|
text = text.lower()
|
||
|
|
elif noise_type < 0.25:
|
||
|
|
text = ''.join(c.upper() if random.random() < 0.2 else c.lower() for c in text)
|
||
|
|
elif noise_type < 0.35:
|
||
|
|
words = text.split()
|
||
|
|
if len(words) > 1:
|
||
|
|
idx = random.randint(0, len(words)-2)
|
||
|
|
words[idx] = words[idx] + words[idx+1]
|
||
|
|
del words[idx+1]
|
||
|
|
text = ' '.join(words)
|
||
|
|
elif noise_type < 0.4:
|
||
|
|
text = re.sub(r' ', ' ', text, count=1)
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
def inject_asr_noise(text, noise_level=0.4):
|
||
|
|
words = text.lower().split()
|
||
|
|
result = []
|
||
|
|
for i, word in enumerate(words):
|
||
|
|
if random.random() < 0.05 * noise_level:
|
||
|
|
continue
|
||
|
|
if word in PHONETIC_NOISE and random.random() < noise_level:
|
||
|
|
result.append(random.choice(PHONETIC_NOISE[word]))
|
||
|
|
elif random.random() < noise_level * 0.3 and len(word) > 3:
|
||
|
|
noise_ops = ['swap', 'delete', 'insert', 'replace']
|
||
|
|
op = random.choice(noise_ops)
|
||
|
|
w = list(word)
|
||
|
|
if op == 'swap' and len(w) > 2:
|
||
|
|
idx = random.randint(0, len(w)-2)
|
||
|
|
w[idx], w[idx+1] = w[idx+1], w[idx]
|
||
|
|
elif op == 'delete' and len(w) > 2:
|
||
|
|
idx = random.randint(0, len(w)-1)
|
||
|
|
del w[idx]
|
||
|
|
elif op == 'insert':
|
||
|
|
idx = random.randint(0, len(w))
|
||
|
|
w.insert(idx, random.choice('abcdefghijklmnopqrstuvwxyz'))
|
||
|
|
elif op == 'replace':
|
||
|
|
idx = random.randint(0, len(w)-1)
|
||
|
|
w[idx] = random.choice('abcdefghijklmnopqrstuvwxyz')
|
||
|
|
result.append(''.join(w))
|
||
|
|
else:
|
||
|
|
result.append(word)
|
||
|
|
if random.random() < 0.08 * noise_level:
|
||
|
|
filler = random.choice(BG_NOISE_INSERTIONS)
|
||
|
|
if filler:
|
||
|
|
result.append(filler)
|
||
|
|
if random.random() < 0.1 * noise_level and len(result) > 0:
|
||
|
|
idx = random.randint(0, len(result)-1)
|
||
|
|
result.insert(idx, result[idx])
|
||
|
|
text = ' '.join(result)
|
||
|
|
text = apply_formatting_noise(text)
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
def generate_variations(phrase, num_variations=5):
|
||
|
|
variations = []
|
||
|
|
for _ in range(num_variations):
|
||
|
|
noise_level = random.uniform(0.2, 0.8)
|
||
|
|
noisy = inject_asr_noise(phrase, noise_level)
|
||
|
|
if noisy.strip() and noisy != phrase.lower():
|
||
|
|
variations.append(noisy)
|
||
|
|
variations.append(phrase.lower())
|
||
|
|
variations.append(phrase)
|
||
|
|
return variations
|
||
|
|
|
||
|
|
|
||
|
|
SYSTEM_PROMPT = """You are a game command interpreter. The user's input is noisy speech-to-text from a microphone in a gaming environment. Your job is to identify the intended game command from the noisy input.
|
||
|
|
|
||
|
|
Valid commands: {commands}
|
||
|
|
|
||
|
|
Respond with ONLY a JSON object: {{"command": "<CommandName>"}}
|
||
|
|
If the input doesn't match any command, respond: {{"command": "Unknown"}}"""
|
||
|
|
|
||
|
|
|
||
|
|
def get_system_prompt():
|
||
|
|
command_list = ", ".join(sorted(GAME_COMMANDS.keys()))
|
||
|
|
return SYSTEM_PROMPT.format(commands=command_list)
|
||
|
|
|
||
|
|
|
||
|
|
def generate_dataset(num_samples_per_command=100, include_negatives=True):
|
||
|
|
samples = []
|
||
|
|
system_msg = get_system_prompt()
|
||
|
|
for cmd_name, cmd_info in GAME_COMMANDS.items():
|
||
|
|
for phrase in cmd_info["canonical_phrases"]:
|
||
|
|
num_vars = max(3, num_samples_per_command // len(cmd_info["canonical_phrases"]))
|
||
|
|
variations = generate_variations(phrase, num_variations=num_vars)
|
||
|
|
for noisy_text in variations:
|
||
|
|
if not noisy_text.strip():
|
||
|
|
continue
|
||
|
|
sample = {
|
||
|
|
"messages": [
|
||
|
|
{"role": "system", "content": system_msg},
|
||
|
|
{"role": "user", "content": noisy_text},
|
||
|
|
{"role": "assistant", "content": json.dumps({"command": cmd_name})}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
samples.append(sample)
|
||
|
|
if include_negatives:
|
||
|
|
negative_phrases = [
|
||
|
|
"what time is it", "hello how are you", "this game is awesome",
|
||
|
|
"I need to go to the bathroom", "what's the score", "can you hear me",
|
||
|
|
"testing testing one two three", "hold on let me think", "that was close",
|
||
|
|
"nice shot dude", "where are we going", "I don't know what to do",
|
||
|
|
"this is so hard", "let's take a different route", "watch out behind you",
|
||
|
|
"I think we should go left", "what level are you",
|
||
|
|
"how much health do you have", "we need more players",
|
||
|
|
"good game everyone", "random noise blah blah", "shshhshshs",
|
||
|
|
"mmmhmm yeah", "no no no wait", "okay okay I got it",
|
||
|
|
]
|
||
|
|
for phrase in negative_phrases:
|
||
|
|
for _ in range(8):
|
||
|
|
noisy = inject_asr_noise(phrase, noise_level=random.uniform(0.1, 0.5))
|
||
|
|
sample = {
|
||
|
|
"messages": [
|
||
|
|
{"role": "system", "content": system_msg},
|
||
|
|
{"role": "user", "content": noisy},
|
||
|
|
{"role": "assistant", "content": json.dumps({"command": "Unknown"})}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
samples.append(sample)
|
||
|
|
random.shuffle(samples)
|
||
|
|
return samples
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
random.seed(42)
|
||
|
|
print("Generating training dataset...")
|
||
|
|
train_samples = generate_dataset(num_samples_per_command=120)
|
||
|
|
print(f" Training samples: {len(train_samples)}")
|
||
|
|
|
||
|
|
random.seed(123)
|
||
|
|
print("Generating test dataset...")
|
||
|
|
test_samples = generate_dataset(num_samples_per_command=20)
|
||
|
|
print(f" Test samples: {len(test_samples)}")
|
||
|
|
|
||
|
|
train_ds = Dataset.from_list(train_samples)
|
||
|
|
test_ds = Dataset.from_list(test_samples)
|
||
|
|
dataset_dict = DatasetDict({"train": train_ds, "test": test_ds})
|
||
|
|
|
||
|
|
print(f"\nPushing to Hub...")
|
||
|
|
dataset_dict.push_to_hub("Marco333/game-commands-noisy-stt", private=False)
|
||
|
|
print("Done!")
|