Day 11: Golden Benchmark Suites for Skills

[
  {
    "id": "TRIG-001",
    "prompt": "Write isolated unit tests for SettlementLedgerService with MockK",
    "expected_trigger": true,
    "category": "TRUE_POSITIVE",
    "rationale": "Clear service-layer unit testing request requiring MockK."
  },
  {
    "id": "TRIG-002",
    "prompt": "Test PaymentGatewayClient using Testcontainers and real PostgreSQL database",
    "expected_trigger": false,
    "category": "TRUE_NEGATIVE_INTEGRATION",
    "rationale": "Explicitly requests Testcontainers and database; must be pruned by negative guard."
  },
  {
    "id": "TRIG-003",
    "prompt": "Create unit tests for FxExecutionEngine mocking suspend methods with coEvery",
    "expected_trigger": true,
    "category": "TRUE_POSITIVE",
    "rationale": "Domain logic suspend function mocking with MockK."
  },
  {
    "id": "TRIG-004",
    "prompt": "Write @SpringBootTest integration tests with MockMvc for UserController",
    "expected_trigger": false,
    "category": "TRUE_NEGATIVE_MVC",
    "rationale": "Requests @SpringBootTest and HTTP mocking; violates pure JVM boundary."
  },
  {
    "id": "TRIG-005",
    "prompt": "Write tests for BillingService verifying transaction rollback in case of error",
    "expected_trigger": true,
    "category": "BORDERLINE_SERVICE_UNIT",
    "rationale": "Unit testing business exception rollback behavior without database."
  },
  {
    "id": "TRIG-006",
    "prompt": "Setup Cypress end-to-end browser test for trading checkout flow",
    "expected_trigger": false,
    "category": "TRUE_NEGATIVE_E2E",
    "rationale": "Frontend E2E test; completely unrelated to backend JVM unit testing."
  }
]
          

#!/usr/bin/env python3
"""
run-skill-evals.py
Standard: agentskills Standard v1.0 (Phase 4 Verification & Distribution)
Evaluates Semantic Trigger Precision/Recall and compiles a comprehensive Confusion Matrix.
Exits with 0 on passing all gates, 1 on regression failure.
"""
import sys
import os
import json
import re
from pathlib import Path

# // [TODAY'S LESSON PLUGS IN HERE: DETERMINISTIC GATING & CONFUSION MATRIX]
def load_skill_description(skill_path: Path) -> str:
    """Extracts description from SKILL.md YAML frontmatter."""
    content = skill_path.read_text(encoding="utf-8")
    match = re.search(r"^---\s*([\s\S]*?)\s*---", content, re.MULTILINE)
    if not match:
        raise ValueError("Invalid SKILL.md: Frontmatter not detected.")
    
    fm = match.group(1)
    desc_match = re.search(r"description:\s*>?-?\s*([\s\S]*?)(?=\n[a-zA-Z0-9_-]+:|$)", fm)
    if not desc_match:
        raise ValueError("No description field found in frontmatter.")
    return " ".join(desc_match.group(1).split())

def evaluate_semantic_routing(description: str, prompt: str) -> bool:
    """
    Deterministic Dual-Barrier Routing Simulator.
    Models Host Agent embedding match:
    1. Positive Intent Activation
    2. Negative Guard Pruning
    """
    p_lower = prompt.lower()
    d_lower = description.lower()

    # Stage 1: Negative Boundary Guard (Immediate Pruning)
    negative_signals = [
        "testcontainers", "postgresql", "database", "datajpatest", 
        "springboottest", "mockmvc", "cypress", "playwright", "end to end", "e2e"
    ]
    for neg in negative_signals:
        if neg in p_lower and "do not use when:" in d_lower:
            return False

    # Stage 2: Positive Intent Match
    positive_signals = ["unit test", "mockk", "coevery", "isolated", "service", "domain"]
    match_count = sum(1 for pos in positive_signals if pos in p_lower)

    return match_count >= 1

def run_benchmark(dataset_path: Path, skill_path: Path) -> dict:
    description = load_skill_description(skill_path)
    test_cases = json.loads(dataset_path.read_text(encoding="utf-8"))

    tp = fp = tn = fn = 0
    failures = []

    for tc in test_cases:
        predicted = evaluate_semantic_routing(description, tc["prompt"])
        expected = tc["expected_trigger"]

        if predicted and expected:
            tp += 1
        elif predicted and not expected:
            fp += 1
            failures.append({"id": tc["id"], "type": "FALSE_POSITIVE", "prompt": tc["prompt"]})
        elif not predicted and not expected:
            tn += 1
        elif not predicted and expected:
            fn += 1
            failures.append({"id": tc["id"], "type": "FALSE_NEGATIVE", "prompt": tc["prompt"]})

    total = len(test_cases)
    precision = (tp / (tp + fp)) * 100 if (tp + fp) > 0 else 0.0
    recall = (tp / (tp + fn)) * 100 if (tp + fn) > 0 else 0.0
    accuracy = ((tp + tn) / total) * 100 if total > 0 else 0.0
    f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0

    return {
        "metrics": {
            "total_cases": total,
            "tp": tp, "fp": fp, "tn": tn, "fn": fn,
            "precision": round(precision, 2),
            "recall": round(recall, 2),
            "accuracy": round(accuracy, 2),
            "f1": round(f1, 2)
        },
        "failures": failures
    }

def main():
    root = Path(__file__).resolve().parent
    dataset = root / "datasets" / "trigger-golden-suite.json"
    skill = root.parent / "skills" / "spring-boot-unit-tester" / "SKILL.md"

    if not dataset.exists() or not skill.exists():
        sys.stderr.write(f"Error: Required paths missing. Dataset: {dataset.exists()}, Skill: {skill.exists()}\n")
        sys.exit(1)

    results = run_benchmark(dataset, skill)
    m = results["metrics"]

    print("================================================================================")
    print("📊 DETERMINISTIC AGENT SKILL BENCHMARK REPORT (Day 11)")
    print("================================================================================")
    print(f"Total Test Cases: {m['total_cases']} | Accuracy: {m['accuracy']}% | F1-Score: {m['f1']}%")
    print(f"Confusion Matrix: [TP: {m['tp']} | FP: {m['fp']} | TN: {m['tn']} | FN: {m['fn']}]")
    print(f"Trigger Precision: {m['precision']}% (Threshold Gate: >= 95.0%)")
    print(f"Trigger Recall:    {m['recall']}% (Threshold Gate: >= 90.0%)")
    print("--------------------------------------------------------------------------------")

    passed = True
    if m["precision"] < 95.0:
        print(f"❌ REGRESSION GATE FAILED: Precision {m['precision']}% is below 95.0% threshold.")
        passed = False
    if m["recall"] < 90.0:
        print(f"❌ REGRESSION GATE FAILED: Recall {m['recall']}% is below 90.0% threshold.")
        passed = False

    if results["failures"]:
        print("\n🔍 FAILED EVALUATION TEST CASES:")
        for f in results["failures"]:
            print(f"  - [{f['id']}] {f['type']}: '{f['prompt']}'")

    if not passed:
        print("\n🛑 PULL REQUEST REJECTED: Semantic trigger regressions detected.")
        sys.exit(1)

    print("\n🏆 ALL BENCHMARK GATES PASSED: Ready for enterprise distribution!")
    sys.exit(0)

if __name__ == "__main__":
    main()