27 lines
1.7 KiB
Python
27 lines
1.7 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Quick self-contained eval: runs the GGUF (grammar-constrained) on built-in test passages and
|
||
|
|
# reports JSON-validity, sarcasm handling, and contradiction detection. pip install llama-cpp-python
|
||
|
|
import glob, json, os
|
||
|
|
from llama_cpp import Llama, LlamaGrammar
|
||
|
|
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
|
||
|
|
GGUF = (glob.glob(os.path.join(ROOT, "*.gguf")) + glob.glob(os.path.join(HERE, "*.gguf")))[0]
|
||
|
|
PROMPT = open(os.path.join(ROOT, "prompt.txt"), encoding="utf-8").read()
|
||
|
|
GRAMMAR = LlamaGrammar.from_string(open(os.path.join(ROOT, "claim.gbnf"), encoding="utf-8").read())
|
||
|
|
CASES = [
|
||
|
|
("The mayor said crime fell last year. Hours later the police chief said it actually rose.", "contradiction"),
|
||
|
|
("What an incredible update, my phone now dies by noon. The engineers really outdid themselves.", "sarcasm"),
|
||
|
|
("On 2024-03-12 the central bank cut rates to 4.5%, citing easing inflation.", "fact"),
|
||
|
|
]
|
||
|
|
llm = Llama(model_path=GGUF, n_ctx=4096, n_gpu_layers=-1, verbose=False)
|
||
|
|
ok = con = sar = nclaims = 0
|
||
|
|
for text, tag in CASES:
|
||
|
|
o = llm.create_chat_completion(messages=[{"role": "user", "content": PROMPT + text}],
|
||
|
|
grammar=GRAMMAR, temperature=0.0, max_tokens=768)
|
||
|
|
try:
|
||
|
|
d = json.loads(o["choices"][0]["message"]["content"]); ok += 1; nclaims += len(d.get("claims", []))
|
||
|
|
if tag == "contradiction" and d.get("contradictions"): con += 1
|
||
|
|
if tag == "sarcasm" and any(c.get("stance") == "ironic" for c in d.get("claims", [])): sar += 1
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
print(f"valid JSON: {ok}/{len(CASES)} | avg claims: {nclaims/max(1,ok):.1f} | contradiction: {con}/1 | sarcasm: {sar}/1")
|