On this page
Compare agent versions
Compare a candidate publication with a baseline using existing runs.
Operation ID: comparisons.create.
Request
POST /api/v1/comparisons
- TypeScript SDK:
client.comparisons.create(path arguments usepath). - MCP tool:
sdk_client_callwithoperationId: "comparisons.create"(path arguments useinput.path). Legacy alias:comparisons_create. Connect to your authenticated/mcpendpoint with an API key for the same environment. - Minimum API key role: member.
- Idempotency: Required. Send
Idempotency-Key(TypeScript SDK:idempotencyKey) and keep it for retries.
Read resource IDs from earlier responses and use a key for the same environment.
Request example
Choose two existing publications and eligible recorded runs from the same agent. Use rubric run-pairwise version 1, completed source runs with retained transcripts, and an environment with test admission and an enabled comparison judge. Poll comparisons.get and inspect its cases. A created comparison is not a passed evaluation.
Set CUMULUS_BASE_URL to your workspace API URL and CUMULUS_API_KEY to a key for that environment. Set IDEMPOTENCY_KEY to a new UUID for this action and keep it for retries. Example values are synthetic.
cURL
curl --fail-with-body -sS -X POST "${CUMULUS_BASE_URL%/}/api/v1/comparisons" \
-H "Authorization: Bearer $CUMULUS_API_KEY" \
-H "Idempotency-Key: $IDEMPOTENCY_KEY" \
-H "Content-Type: application/json" \
--data '{
"agent_id": "service-guide",
"baseline_version": 0,
"candidate_version": 1,
"case_run_ids": [
"33333333-3333-4333-8333-333333333333"
],
"rubric": {
"id": "run-pairwise",
"version": 1
}
}'
TypeScript
import { Cumulus } from "@cumulus/sdk";
const baseUrl = process.env.CUMULUS_BASE_URL;
const apiKey = process.env.CUMULUS_API_KEY;
if (!baseUrl || !apiKey) throw new Error("Set CUMULUS_BASE_URL and CUMULUS_API_KEY");
const client = new Cumulus({ baseUrl, apiKey });
const idempotencyKey = process.env.IDEMPOTENCY_KEY;
if (!idempotencyKey) throw new Error("Set IDEMPOTENCY_KEY for this request");
const response = await client.comparisons.create({
"body": {
"agent_id": "service-guide",
"baseline_version": 0,
"candidate_version": 1,
"case_run_ids": [
"33333333-3333-4333-8333-333333333333"
],
"rubric": {
"id": "run-pairwise",
"version": 1
}
},
idempotencyKey,
});
Input schema
Path, query and header parameters
Idempotency-Key
header parameter · Required.
Type: string. minLength: 1. maxLength: 256.
JSON request body
| Field | Type or values | Required | Description and limits |
|---|---|---|---|
agent_id | string | Required | Agent identifier in the environment selected by your API key. minLength: 1. maxLength: 256. pattern: ^[A-Za-z0-9][A-Za-z0-9_.:-]*$. |
baseline_version | integer | Required | At least 0. At most 2147483647. |
candidate_version | integer | Required | At least 0. At most 2147483647. |
case_run_ids | array of string (uuid) | Required | minItems: 1. maxItems: 200. |
rubric | object | Required | — |
rubric.id | string | Required | minLength: 1. maxLength: 128. |
rubric.version | integer | Required | Greater than 0. At most 9007199254740991. |
Complete OpenAPI schema
{
"parameters": [
{
"name": "Idempotency-Key",
"in": "header",
"required": true,
"schema": {
"type": "string",
"minLength": 1,
"maxLength": 256
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ComparisonsCreateRequest"
}
}
}
}
}
Responses
HTTP 202
Compare a candidate publication against a baseline on existing runs
application/json
| Field | Type or values | Required | Description and limits |
|---|---|---|---|
owner_id | string | Required | minLength: 1. |
environment | string | Required | minLength: 1. |
tenant_id | string (uuid) | Required | pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$. |
comparison_id | string (uuid) | Required | pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$. |
agent_id | string | Required | Agent identifier in the environment selected by your API key. minLength: 1. maxLength: 256. pattern: ^[A-Za-z0-9][A-Za-z0-9_.:-]*$. |
baseline | object | Required | — |
baseline.agent_id | string | Required | Agent identifier in the environment selected by your API key. minLength: 1. maxLength: 256. pattern: ^[A-Za-z0-9][A-Za-z0-9_.:-]*$. |
baseline.version | integer | Required | At least 0. At most 2147483647. |
baseline.digest | string | Required | pattern: ^sha256:[a-f0-9]{64}$. |
candidate | object | Required | — |
candidate.agent_id | string | Required | Agent identifier in the environment selected by your API key. minLength: 1. maxLength: 256. pattern: ^[A-Za-z0-9][A-Za-z0-9_.:-]*$. |
candidate.version | integer | Required | At least 0. At most 2147483647. |
candidate.digest | string | Required | pattern: ^sha256:[a-f0-9]{64}$. |
case_run_ids | array of string (uuid) | Required | minItems: 1. maxItems: 200. |
state | "queued" · "running" · "completed" · "failed" · "uncertain" | Required | — |
summary | object or null | Required | — |
summary.cases | integer | Required in a non-null parent | At least -9007199254740991. At most 9007199254740991. |
summary.candidate_wins | integer | Required in a non-null parent | At least -9007199254740991. At most 9007199254740991. |
summary.baseline_wins | integer | Required in a non-null parent | At least -9007199254740991. At most 9007199254740991. |
summary.ties | integer | Required in a non-null parent | At least -9007199254740991. At most 9007199254740991. |
summary.unscored | integer | Required in a non-null parent | At least -9007199254740991. At most 9007199254740991. |
rubric | object | Required | — |
rubric.id | string | Required | minLength: 1. maxLength: 128. |
rubric.version | integer | Required | Greater than 0. At most 9007199254740991. |
judge_model | object | Required | — |
judge_model.model_id | string | Required | minLength: 1. pattern: \S. |
judge_model.provider | string | Required | minLength: 1. pattern: \S. |
judge_model.tier | "frontier" · "cumulus-served" · "customer-gpu" | Required | — |
error_code | string or null | Required | maxLength: 128. |
finished_at | string (date-time) or null | Required | pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:.\d+)?(?:Z))$. |
created_by | object | Required | — |
| created_by · variant 1 | object | — | Fields for this schema branch. |
created_by.owner_id | string | Required | minLength: 1. |
created_by.environment | string | Required | minLength: 1. |
created_by.kind | "user" | Required | — |
created_by.id | string (uuid) | Required | pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$. |
| created_by · variant 2 | object | — | Fields for this schema branch. |
created_by.owner_id | string | Required | minLength: 1. |
created_by.environment | string | Required | minLength: 1. |
created_by.kind | "api_key" | Required | — |
created_by.id | string (uuid) | Required | pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$. |
| created_by · variant 3 | object | — | Fields for this schema branch. |
created_by.owner_id | string | Required | minLength: 1. |
created_by.environment | string | Required | minLength: 1. |
created_by.kind | "system" | Required | — |
created_by.id | string | Required | minLength: 1. maxLength: 256. |
created_at | string (date-time) | Required | pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:.\d+)?(?:Z))$. |
Error responses
| HTTP | Description |
|---|---|
| 4XX | Typed error (ApiError) |
| 5XX | Typed error (ApiError) |
Errors use the shared error envelope. Inspect error.code, retryable, outcome and details before retrying.
Complete OpenAPI schema
{
"202": {
"description": "Compare a candidate publication against a baseline on existing runs",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Comparison"
}
}
}
},
"4XX": {
"description": "Typed error (ApiError)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiError"
}
}
}
},
"5XX": {
"description": "Typed error (ApiError)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiError"
}
}
}
}
}
Referenced schemas
Expand the complete OpenAPI definitions for integrations that generate their own clients.
Complete OpenAPI schema
{
"components": {
"schemas": {
"ComparisonsCreateRequest": {
"type": "object",
"properties": {
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"baseline_version": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647
},
"candidate_version": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647
},
"case_run_ids": {
"minItems": 1,
"maxItems": 200,
"type": "array",
"items": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
}
},
"rubric": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1,
"maxLength": 128
},
"version": {
"type": "integer",
"minimum": 0,
"exclusiveMinimum": true,
"maximum": 9007199254740991
}
},
"required": [
"id",
"version"
],
"additionalProperties": false
}
},
"required": [
"agent_id",
"baseline_version",
"candidate_version",
"case_run_ids",
"rubric"
],
"additionalProperties": false
},
"Comparison": {
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"tenant_id": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
},
"comparison_id": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
},
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"baseline": {
"$ref": "#/components/schemas/PublicationRef"
},
"candidate": {
"$ref": "#/components/schemas/PublicationRef"
},
"case_run_ids": {
"minItems": 1,
"maxItems": 200,
"type": "array",
"items": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
}
},
"state": {
"type": "string",
"enum": [
"queued",
"running",
"completed",
"failed",
"uncertain"
]
},
"summary": {
"nullable": true,
"type": "object",
"properties": {
"cases": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"candidate_wins": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"baseline_wins": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"ties": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"unscored": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
}
},
"required": [
"cases",
"candidate_wins",
"baseline_wins",
"ties",
"unscored"
],
"additionalProperties": false
},
"rubric": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1,
"maxLength": 128
},
"version": {
"type": "integer",
"minimum": 0,
"exclusiveMinimum": true,
"maximum": 9007199254740991
}
},
"required": [
"id",
"version"
],
"additionalProperties": false
},
"judge_model": {
"$ref": "#/components/schemas/ModelIdentity"
},
"error_code": {
"nullable": true,
"type": "string",
"maxLength": 128
},
"finished_at": {
"nullable": true,
"type": "string",
"format": "date-time",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$"
},
"created_by": {
"$ref": "#/components/schemas/PrincipalRef"
},
"created_at": {
"type": "string",
"format": "date-time",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$"
}
},
"required": [
"owner_id",
"environment",
"tenant_id",
"comparison_id",
"agent_id",
"baseline",
"candidate",
"case_run_ids",
"state",
"summary",
"rubric",
"judge_model",
"error_code",
"finished_at",
"created_by",
"created_at"
],
"additionalProperties": false
},
"PublicationRef": {
"type": "object",
"properties": {
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"version": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647
},
"digest": {
"type": "string",
"pattern": "^sha256:[a-f0-9]{64}$"
}
},
"required": [
"agent_id",
"version",
"digest"
],
"additionalProperties": false
},
"ModelIdentity": {
"type": "object",
"properties": {
"model_id": {
"type": "string",
"minLength": 1,
"pattern": "\\S"
},
"provider": {
"type": "string",
"minLength": 1,
"pattern": "\\S"
},
"tier": {
"type": "string",
"enum": [
"frontier",
"cumulus-served",
"customer-gpu"
]
}
},
"required": [
"model_id",
"provider",
"tier"
],
"additionalProperties": false
},
"PrincipalRef": {
"oneOf": [
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"user"
]
},
"id": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"api_key"
]
},
"id": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"system"
]
},
"id": {
"type": "string",
"minLength": 1,
"maxLength": 256
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
}
]
},
"ApiError": {
"type": "object",
"properties": {
"error": {
"type": "object",
"properties": {
"code": {
"type": "string",
"enum": [
"unauthenticated",
"forbidden",
"not_found",
"conflict",
"invalid_request",
"idempotency_conflict",
"policy_denied",
"capability_unavailable",
"provider_unavailable",
"knowledge_unavailable",
"publication_invalid",
"run_not_active",
"rate_limited",
"outcome_unknown",
"internal"
]
},
"message": {
"type": "string"
},
"retryable": {
"type": "boolean"
},
"outcome": {
"type": "string",
"enum": [
"none",
"unknown"
]
},
"details": {
"type": "object",
"additionalProperties": {}
}
},
"required": [
"code",
"message",
"retryable",
"outcome"
],
"additionalProperties": false
}
},
"required": [
"error"
],
"additionalProperties": false
}
}
}
}
Errors and recovery
See API error handling and Troubleshooting.