API Reference

Results Schema

The complete response object returned by evaluate(). The uq field is present only when runs > 1.

Full response object

EvaluateResult (annotated)

1{
2  // The prompt used for this evaluation
3  "prompt": "A customer says: 'My order has been stuck...'",
4 
5  // Number of model runs performed
6  "runs": 10,
7 
8  // Results for each assertion, in the order provided
9  "assertions": [
10    {
11      // Assertion type
12      "name": "not-toxic",
13 
14      // Whether the output (or the majority of outputs in multi-run) passed
15      "passed": true,
16 
17      // Score returned by the evaluator (null for deterministic assertions)
18      "score": 0.02,
19 
20      // Which channel handled this assertion
21      "channel": "encoder",
22 
23      // Per-run scores, present when runs > 1
24      "runScores": [0.02, 0.01, 0.03, 0.02, 0.01, 0.02, 0.03, 0.01, 0.02, 0.02]
25    },
26    {
27      "name": "max-words",
28      "passed": true,
29      "score": null,
30      "channel": "deterministic",
31      "runScores": null
32    },
33    {
34      "name": "empathy",
35      "passed": false,
36      "score": 0.61,          // mean across all runs
37      "channel": "semantic",
38      "threshold": 0.75,
39      "runScores": [0.84, 0.29, 0.83, 0.31, 0.85, 0.82, 0.30, 0.84, 0.83, 0.28]
40    }
41  ],
42 
43  // Statistical analysis — present when runs > 1
44  "uq": {
45    // Number of distinct behavioral modes detected
46    "modeCount": 2,
47 
48    "modes": [
49      {
50        "id": "A",
51        // Fraction of runs that belong to this mode
52        "frequency": 0.60,
53        // Average assertion score for runs in this mode
54        "avgScore": 0.84,
55        // A representative sample output from this mode
56        "sampleOutput": "I completely understand how frustrating this must be...",
57        // Plain-English description of this mode's characteristic behavior
58        "description": "Warm, empathetic, solution-focused responses that acknowledge the customer's frustration"
59      },
60      {
61        "id": "B",
62        "frequency": 0.40,
63        "avgScore": 0.29,
64        "sampleOutput": "Your order is delayed. I'll send an update in two hours.",
65        "description": "Terse, informational responses that address the issue without emotional acknowledgment"
66      }
67    ],
68 
69    // Fraction of runs that passed the semantic assertion threshold
70    "passRate": 0.60,
71 
72    // Wilson 95% confidence interval on passRate
73    "ci95": [0.30, 0.84],
74 
75    // Mean score across all runs
76    "meanScore": 0.61,
77 
78    // Bootstrap 95% CI on the mean score
79    "scoreCI95": [0.41, 0.79],
80 
81    // Comparison against the stored baseline, if one exists
82    "baselineComparison": {
83      // The version tag of the baseline used
84      "baselineVersion": "v1.3.0",
85      "baselinePassRate": 0.90,
86      "currentPassRate": 0.60,
87 
88      // Signed difference (current - baseline)
89      "shift": -0.30,
90 
91      // KS test p-value
92      "pValue": 0.003,
93 
94      // true if p < 0.05
95      "significant": true,
96 
97      // "regression", "improvement", or "neutral"
98      "direction": "regression"
99    }
100  },
101 
102  // Plain-English explanation of findings — present when runs > 1
103  "diagnostic": "A behavioral regression was introduced. Your AI previously had one behavioral mode (warm, empathetic). It now has two distinct modes, with Mode B (terse, transactional) occurring 40% of the time. The distribution shift is statistically significant (p=0.003). Most likely cause: the 'Be concise' instruction added in the latest system prompt competes with the existing empathy instruction. Suggested fix: reframe conciseness as subordinate to empathy.",
104 
105  // ISO 8601 timestamp
106  "evaluatedAt": "2026-03-03T09:42:01Z",
107 
108  // Your version tag, if provided
109  "version": "v1.4.0"
110}

TypeScript types

types.ts

1interface EvaluateResult {
2  prompt: string
3  runs: number
4  assertions: AssertionResult[]
5  uq?: UQResult          // present when runs > 1
6  diagnostic?: string    // present when runs > 1
7  evaluatedAt: string
8  version?: string
9}
10 
11interface AssertionResult {
12  name: string
13  passed: boolean
14  score: number | null
15  channel: "deterministic" | "encoder" | "semantic"
16  threshold?: number
17  runScores: number[] | null
18}
19 
20interface UQResult {
21  modeCount: number
22  modes: Mode[]
23  passRate: number
24  ci95: [number, number]
25  meanScore: number
26  scoreCI95: [number, number]
27  baselineComparison?: BaselineComparison
28}
29 
30interface Mode {
31  id: string
32  frequency: number
33  avgScore: number
34  sampleOutput: string
35  description: string
36}
37 
38interface BaselineComparison {
39  baselineVersion: string
40  baselinePassRate: number
41  currentPassRate: number
42  shift: number
43  pValue: number
44  significant: boolean
45  direction: "regression" | "improvement" | "neutral"
46}