On this page
test_runs.create
Run test suites against one publication version (201; an idempotent replay returns 200)
Request
POST /api/v1/test-runs
- MCP:
test_runs_create - SDK:
client.test_runs.create(SDK path arguments usepath; MCP usesparams). - Minimum API key role: member.
- Idempotency: required_header. Send an
Idempotency-Keyheader; MCP usesidempotency_key. Reuse it after an unknown outcome.
Body {agent_id, version (a published version, live or test), suite_ids}; optional case_ids and trials. Poll test_runs_get until state is completed or cancelled, then test_runs_results for each case's verdict and assertion verdicts, and test_runs_trial for one trial's transcript. test_runs_estimate shows trials, cost and minutes first.
Input schema
json
{
"parameters": [
{
"name": "Idempotency-Key",
"in": "header",
"required": true,
"schema": {
"type": "string",
"minLength": 1,
"maxLength": 256
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestRunCreateRequest"
}
}
}
}
}
Responses
json
{
"200": {
"description": "Run test suites against one publication version (201; an idempotent replay returns 200)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestRun"
}
}
}
},
"201": {
"description": "Run test suites against one publication version (201; an idempotent replay returns 200)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestRun"
}
}
}
},
"4XX": {
"description": "Typed error (ApiError)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiError"
}
}
}
},
"5XX": {
"description": "Typed error (ApiError)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiError"
}
}
}
}
}
Referenced schemas
json
{
"components": {
"schemas": {
"TestRunCreateRequest": {
"type": "object",
"properties": {
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"version": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647
},
"suite_ids": {
"type": "array",
"items": {
"type": "string",
"format": "uuid"
},
"minItems": 1,
"maxItems": 10
},
"case_ids": {
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[a-z0-9][a-z0-9_-]*$"
},
"minItems": 1,
"maxItems": 100
},
"trials": {
"type": "integer",
"minimum": 1,
"maximum": 10
},
"baseline_test_run_id": {
"type": "string",
"format": "uuid"
}
},
"required": [
"agent_id",
"version",
"suite_ids"
],
"additionalProperties": false
},
"TestRun": {
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"tenant_id": {
"type": "string",
"format": "uuid"
},
"test_run_id": {
"type": "string",
"format": "uuid"
},
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"publication": {
"$ref": "#/components/schemas/PublicationRef"
},
"purpose": {
"type": "string",
"enum": [
"manual",
"publish_gate",
"schedule",
"release"
]
},
"suites": {
"type": "array",
"items": {
"type": "object",
"properties": {
"suite_id": {
"type": "string",
"format": "uuid"
},
"revision": {
"type": "integer",
"exclusiveMinimum": true,
"minimum": 0,
"maximum": 2147483647
},
"digest": {
"type": "string",
"pattern": "^sha256:[a-f0-9]{64}$"
}
},
"required": [
"suite_id",
"revision",
"digest"
],
"additionalProperties": false
},
"minItems": 1,
"maxItems": 10
},
"state": {
"type": "string",
"enum": [
"queued",
"running",
"completed",
"cancelled"
]
},
"verdict": {
"type": "string",
"enum": [
"passed",
"failed",
"errored"
],
"nullable": true
},
"progress": {
"type": "object",
"properties": {
"trials": {
"type": "integer",
"minimum": 0
},
"admitted": {
"type": "integer",
"minimum": 0
},
"settled": {
"type": "integer",
"minimum": 0
}
},
"required": [
"trials",
"admitted",
"settled"
],
"additionalProperties": false
},
"summary": {
"type": "object",
"properties": {
"cases": {
"type": "integer",
"minimum": 0
},
"passed": {
"type": "integer",
"minimum": 0
},
"flaky_passed": {
"type": "integer",
"minimum": 0
},
"failed": {
"type": "integer",
"minimum": 0
},
"errored": {
"type": "integer",
"minimum": 0
},
"invalid": {
"type": "integer",
"minimum": 0
},
"trials": {
"type": "integer",
"minimum": 0
},
"regressed": {
"type": "integer",
"minimum": 0
},
"fixed": {
"type": "integer",
"minimum": 0
},
"guard_events": {
"type": "integer",
"minimum": 0
}
},
"required": [
"cases",
"passed",
"flaky_passed",
"failed",
"errored",
"invalid",
"trials",
"regressed",
"fixed",
"guard_events"
],
"additionalProperties": false,
"nullable": true
},
"baseline_test_run_id": {
"allOf": [
{
"type": "string",
"format": "uuid"
}
],
"nullable": true
},
"judge_model": {
"allOf": [
{
"$ref": "#/components/schemas/ModelIdentity"
}
],
"nullable": true
},
"cost": {
"type": "object",
"properties": {
"estimated_usd": {
"type": "number",
"minimum": 0
},
"measured_usd": {
"type": "number",
"minimum": 0,
"nullable": true
}
},
"required": [
"estimated_usd",
"measured_usd"
],
"additionalProperties": false
},
"created_by": {
"$ref": "#/components/schemas/PrincipalRef"
},
"created_at": {
"type": "string",
"format": "date-time"
},
"finished_at": {
"allOf": [
{
"type": "string",
"format": "date-time"
}
],
"nullable": true
}
},
"required": [
"owner_id",
"environment",
"tenant_id",
"test_run_id",
"agent_id",
"publication",
"purpose",
"suites",
"state",
"verdict",
"progress",
"summary",
"baseline_test_run_id",
"judge_model",
"cost",
"created_by",
"created_at",
"finished_at"
],
"additionalProperties": false
},
"PublicationRef": {
"type": "object",
"properties": {
"agent_id": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"
},
"version": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647
},
"digest": {
"type": "string",
"pattern": "^sha256:[a-f0-9]{64}$"
}
},
"required": [
"agent_id",
"version",
"digest"
],
"additionalProperties": false
},
"ModelIdentity": {
"type": "object",
"properties": {
"model_id": {
"type": "string",
"minLength": 1,
"pattern": "\\S"
},
"provider": {
"type": "string",
"minLength": 1,
"pattern": "\\S"
},
"tier": {
"type": "string",
"enum": [
"frontier",
"cumulus-served",
"customer-gpu"
]
}
},
"required": [
"model_id",
"provider",
"tier"
],
"additionalProperties": false
},
"PrincipalRef": {
"anyOf": [
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"user"
]
},
"id": {
"type": "string",
"format": "uuid"
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"api_key"
]
},
"id": {
"type": "string",
"format": "uuid"
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"owner_id": {
"type": "string",
"minLength": 1
},
"environment": {
"type": "string",
"minLength": 1
},
"kind": {
"type": "string",
"enum": [
"system"
]
},
"id": {
"type": "string",
"minLength": 1,
"maxLength": 256
}
},
"required": [
"owner_id",
"environment",
"kind",
"id"
],
"additionalProperties": false
}
]
},
"ApiError": {
"type": "object",
"properties": {
"error": {
"type": "object",
"properties": {
"code": {
"type": "string",
"enum": [
"unauthenticated",
"forbidden",
"not_found",
"conflict",
"invalid_request",
"idempotency_conflict",
"policy_denied",
"capability_unavailable",
"provider_unavailable",
"knowledge_unavailable",
"publication_invalid",
"run_not_active",
"rate_limited",
"outcome_unknown",
"internal"
]
},
"message": {
"type": "string"
},
"retryable": {
"type": "boolean"
},
"outcome": {
"type": "string",
"enum": [
"none",
"unknown"
]
},
"details": {
"type": "object",
"additionalProperties": {}
}
},
"required": [
"code",
"message",
"retryable",
"outcome"
],
"additionalProperties": false
}
},
"required": [
"error"
],
"additionalProperties": false
}
}
}
}
Errors and recovery
See API error handling and Troubleshooting.
Related guides
Content version 83511b8eMarkdown source