agents Review

--- name: agent-evaluation description: Testing and benchmarking LLM agents including behavioral testing, capability assessment, reliability metrics, and production monitoring:where even top agents achieve less than 50% on real-world benchmarks risk: safe source: vibeship-spawner-skills (Apache 2.0) dateadded: 2026-02-27 --- # Agent Evaluation Testing and benchmarking LLM agents including behavioral testing, capability assessment, reliability metrics, and production monitoring:where even top agents achieve less than 50% on real-world benchmarks ## Capabilities - agent-testing - benchmark-design - capability-assessment - reliability-metrics - regression-testing ## Prerequisites - Knowledge: Testing methodologies, Statistical analysis basics, LLM behavior patterns - Skillsrecommended: autonomous-agents, multi-agent-orchestration - Required skills: testing-fundamentals, llm-fundamentals ## Scope - Doesnotcover: Model training evaluation (loss, perplexity), Fairness and bias testing, User experience testing - Boundaries: Focus is agent capability and reliability, Covers functional and behavioral testing ## Ecosystem ### Primary_tools - AgentBench - Multi-environment benchmark for LLM agents (ICLR 2024) - τ-bench (Tau-bench) - Sierra's real-world agent benchmark - ToolEmu - Risky behavior detection for agent tool use - Langsmith - LLM tracing and evaluation platform ### Alternatives - Braintrust - When: Need production monitoring integration LLM evaluation and monitoring - PromptFoo - When: Focus on prompt-level evaluation Prompt testing framework ### Deprecated - Manual testing only ## Patterns ### Statistical Test Evaluation Run tests multiple times and analyze result distributions When to use: Evaluating stochastic agent behavior interface TestResult { testId: string; runId: string; passed: boolean; score: number; // 0-1 for partial credit latencyMs: number; tokensUsed: number; output: string; expectedBehaviors: string[]; actualBehaviors: string[]; } interface StatisticalAnalysis { passRate: number; confidence95: [number, number]; meanScore: number; stdDevScore: number; meanLatency: number; p95Latency: number; behaviorConsistency: number; } class StatisticalEvaluator { private readonly minRuns = 10; private readonly confidenceLevel = 0.95; async evaluateAgent( agent: Agent, testSuite: TestCase[] ): Promise<EvaluationReport> { const results: TestResult[] = []; // Run each test multiple times for (const test of testSuite) { for (let run = 0; run < this.minRuns; run++) { const result = await this.runTest(agent, test, run); results.push(result); } } // Analyze by test const byTest = this.groupByTest(results); const testAnalyses = new Map<string, StatisticalAnalysis>(); for (const [testId, testResults] of byTest) { testAnalyses.set(testId, this.analyzeResults(testResults)); } // Overall analysis const overall = this.analyzeResults(results); return { overall, byTest: testAnalyses, concerns: this.identifyConcerns(testAnalyses), recommendations: this.generateRecommendations(testAnalyses) }; } private analyzeResults(results: TestResult[]): StatisticalAnalysis { const passes = results.filter(r => r.passed); const passRate = passes.length / results.length; // Calculate confidence interval for pass rate const z = 1.96; // 95% confidence const se = Math.sqrt((passRate (1 - passRate)) / results.length); const confidence95: [number, number] = [ Math.max(0, passRate - z se), Math.min(1, passRate + z * se) ]; const scores = results.map(r => r.score); const latencies = results.map(r => r.latencyMs); return { passRate, confidence95, meanScore: this.mean(scores), stdDevScore: this.stdDev(scores), meanLatency: this.mean(latencies), p95Latency: this.percentile(latencies, 95), behaviorConsistency: this.calculateConsistency(results) }; } private calculateConsistency(results: TestResult[]): number { // How consistent are the behaviors across runs? if (results.length < 2) return 1; const behaviorSets = results.map(r => new Set(r.actualBehaviors)); let consistencySum = 0; let comparisons = 0; for (let i = 0; i < behaviorSets.length; i++) { for (let j = i + 1; j < behaviorSets.length; j++) { const intersection = new Set( [...behaviorSets[i]].filter(x => behaviorSets[j].has(x)) ); const union = new Set([...behaviorSets[i], ...behaviorSets[j]]