初始化项目,由ModelHub XC社区提供模型

Model: Luimas/claim-extractor-detective-qwen3b
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-27 03:19:10 +08:00
commit 4787f3f1ed
35 changed files with 3569 additions and 0 deletions

26
scripts/evaluate.py Normal file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python
# Quick self-contained eval: runs the GGUF (grammar-constrained) on built-in test passages and
# reports JSON-validity, sarcasm handling, and contradiction detection. pip install llama-cpp-python
import glob, json, os
from llama_cpp import Llama, LlamaGrammar
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
GGUF = (glob.glob(os.path.join(ROOT, "*.gguf")) + glob.glob(os.path.join(HERE, "*.gguf")))[0]
PROMPT = open(os.path.join(ROOT, "prompt.txt"), encoding="utf-8").read()
GRAMMAR = LlamaGrammar.from_string(open(os.path.join(ROOT, "claim.gbnf"), encoding="utf-8").read())
CASES = [
("The mayor said crime fell last year. Hours later the police chief said it actually rose.", "contradiction"),
("What an incredible update, my phone now dies by noon. The engineers really outdid themselves.", "sarcasm"),
("On 2024-03-12 the central bank cut rates to 4.5%, citing easing inflation.", "fact"),
]
llm = Llama(model_path=GGUF, n_ctx=4096, n_gpu_layers=-1, verbose=False)
ok = con = sar = nclaims = 0
for text, tag in CASES:
o = llm.create_chat_completion(messages=[{"role": "user", "content": PROMPT + text}],
grammar=GRAMMAR, temperature=0.0, max_tokens=768)
try:
d = json.loads(o["choices"][0]["message"]["content"]); ok += 1; nclaims += len(d.get("claims", []))
if tag == "contradiction" and d.get("contradictions"): con += 1
if tag == "sarcasm" and any(c.get("stance") == "ironic" for c in d.get("claims", [])): sar += 1
except Exception:
pass
print(f"valid JSON: {ok}/{len(CASES)} | avg claims: {nclaims/max(1,ok):.1f} | contradiction: {con}/1 | sarcasm: {sar}/1")