62 lines
3.4 KiB
Bash
62 lines
3.4 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# marvy-14B quick validation harness.
|
||
|
|
#
|
||
|
|
# Runs the task-coverage probes (Test 2 in VALIDATION.md) against any
|
||
|
|
# OpenAI-compatible endpoint — LM Studio, mlx_lm server, vLLM, etc. — and prints
|
||
|
|
# each artifact plus a lightweight heuristic PASS/FAIL on domain keywords.
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# bash validate.sh # defaults to LM Studio
|
||
|
|
# BASE_URL=http://localhost:8080/v1 MODEL=marvy-14B bash validate.sh
|
||
|
|
# API_KEY=xxx BASE_URL=https://... MODEL=MainStack/marvy-14B bash validate.sh
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
BASE_URL="${BASE_URL:-http://localhost:1234/v1}" # LM Studio default
|
||
|
|
MODEL="${MODEL:-marvy-14B}"
|
||
|
|
API_KEY="${API_KEY:-lm-studio}"
|
||
|
|
TEMP="${TEMP:-0.4}"
|
||
|
|
MAXTOK="${MAXTOK:-700}"
|
||
|
|
|
||
|
|
SYSTEM="You are a senior ServiceNow delivery consultant. You produce precise, implementation-grade artifacts: business analyses, requirements, solution design documents, user stories with acceptance criteria, test cases, and validation reviews. You favor out-of-the-box capabilities, cite concrete tables/plugins/sys_ids when relevant, and write in clear professional English."
|
||
|
|
|
||
|
|
# probe | expected-keyword-regex (case-insensitive) for a heuristic pass
|
||
|
|
PROMPTS=(
|
||
|
|
"Write a user story with acceptance criteria for auto-escalating P1 incidents that breach a 15-minute response SLA.|as a.*i want.*so that|acceptance|sla"
|
||
|
|
"Draft the Incident Management section of an SDD for a greenfield ITSM implementation. Include assignment rules and SLA design.|assignment|sla|incident"
|
||
|
|
"Extract structured requirements (id, category, priority, target phase, success metric) from: replace email-based access requests with a catalog item routed for manager approval.|priority|requirement|catalog"
|
||
|
|
"Write a test case for the story: Restrict the Assignment Group field on incidents to groups with the itil role.|pre-condition|step|expected|itil"
|
||
|
|
"Validate this requirement against best practice and list follow-up questions: All incidents must auto-close after 3 days.|follow-up|risk|question"
|
||
|
|
)
|
||
|
|
|
||
|
|
command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required (brew install jq)"; exit 1; }
|
||
|
|
|
||
|
|
echo "Endpoint: $BASE_URL Model: $MODEL Temp: $TEMP"
|
||
|
|
echo "============================================================"
|
||
|
|
pass=0; total=0
|
||
|
|
for entry in "${PROMPTS[@]}"; do
|
||
|
|
total=$((total+1))
|
||
|
|
prompt="${entry%%|*}"
|
||
|
|
rest="${entry#*|}"; regex="$rest"
|
||
|
|
payload=$(jq -n --arg m "$MODEL" --arg s "$SYSTEM" --arg p "$prompt" \
|
||
|
|
--argjson t "$TEMP" --argjson mx "$MAXTOK" \
|
||
|
|
'{model:$m,temperature:$t,max_tokens:$mx,messages:[{role:"system",content:$s},{role:"user",content:$p}]}')
|
||
|
|
resp=$(curl -s "$BASE_URL/chat/completions" -H "Content-Type: application/json" \
|
||
|
|
-H "Authorization: Bearer $API_KEY" -d "$payload")
|
||
|
|
content=$(echo "$resp" | jq -r '.choices[0].message.content // .error.message // "<<no response>>"')
|
||
|
|
echo ""
|
||
|
|
echo "### Probe $total: $prompt"
|
||
|
|
echo "------------------------------------------------------------"
|
||
|
|
echo "$content" | head -40
|
||
|
|
if echo "$content" | grep -iqE "$regex"; then
|
||
|
|
echo ">>> heuristic: PASS"
|
||
|
|
pass=$((pass+1))
|
||
|
|
else
|
||
|
|
echo ">>> heuristic: REVIEW (expected pattern not matched: $regex)"
|
||
|
|
fi
|
||
|
|
echo "============================================================"
|
||
|
|
done
|
||
|
|
echo ""
|
||
|
|
echo "Heuristic result: $pass/$total probes matched domain patterns."
|
||
|
|
echo "Pass threshold: >= 4/5 with implementation-grade, ServiceNow-specific content."
|
||
|
|
echo "Note: heuristics are a sanity check — read the outputs to judge true quality."
|