Source: https://docs.cumuluslabs.io/reference/test_suites.edit Content version: 29ad4c0a # test_suites.edit Edit a test suite (rename, upsert, remove or reorder cases) as a compare-and-set on its revision ## Request {#request} `PATCH /api/v1/test-suites/{suite_id}` - MCP: `test_suites_edit` - SDK: `client.test_suites.edit` (SDK path arguments use `path`; MCP uses `params`). - Minimum API key role: **member**. - Idempotency: **natural**. Use the request schema below. Read IDs from earlier operation results. ## Input schema {#input-schema} ```json { "parameters": [ { "name": "suite_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TestSuitesEditRequest" } } } } } ``` ## Responses {#responses} ```json { "200": { "description": "Edit a test suite (rename, upsert, remove or reorder cases) as a compare-and-set on its revision", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TestSuiteView" } } } }, "4XX": { "description": "Typed error (ApiError)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } } }, "5XX": { "description": "Typed error (ApiError)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } } } } ``` ## Referenced schemas {#schemas} ```json { "components": { "schemas": { "TestSuitesEditRequest": { "type": "object", "properties": { "expected_revision": { "type": "integer", "exclusiveMinimum": true, "minimum": 0, "maximum": 2147483647 }, "edits": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "op": { "type": "string", "enum": [ "set_name" ] }, "name": { "type": "string", "minLength": 1, "maxLength": 200 } }, "required": [ "op", "name" ], "additionalProperties": false }, { "type": "object", "properties": { "op": { "type": "string", "enum": [ "upsert_case" ] }, "case": { "$ref": "#/components/schemas/TestCase" } }, "required": [ "op", "case" ], "additionalProperties": false }, { "type": "object", "properties": { "op": { "type": "string", "enum": [ "remove_case" ] }, "case_id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z0-9][a-z0-9_-]*$" } }, "required": [ "op", "case_id" ], "additionalProperties": false }, { "type": "object", "properties": { "op": { "type": "string", "enum": [ "reorder_cases" ] }, "case_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z0-9][a-z0-9_-]*$" }, "maxItems": 100 } }, "required": [ "op", "case_ids" ], "additionalProperties": false }, { "type": "object", "properties": { "op": { "type": "string", "enum": [ "set_source" ] }, "source": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "selector" ] }, "window_days": { "type": "integer", "minimum": 1, "maximum": 90 }, "versions": { "type": "array", "items": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "maxItems": 20 }, "contact": { "type": "array", "items": { "type": "string", "enum": [ "human", "voicemail", "ivr", "unknown" ] }, "minItems": 1, "default": [ "human" ] }, "dispositions": { "type": "array", "items": { "type": "string", "pattern": "^[a-z][a-z0-9_.:-]{0,127}$" }, "maxItems": 32 }, "min_caller_turns": { "type": "integer", "minimum": 1, "default": 1 }, "sample": { "type": "string", "enum": [ "latest", "stratified_by_disposition" ], "default": "latest" }, "limit": { "type": "integer", "minimum": 1, "maximum": 200 } }, "required": [ "kind", "window_days", "limit" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "runs" ] }, "run_ids": { "type": "array", "items": { "type": "string", "format": "uuid" }, "minItems": 1, "maxItems": 200 } }, "required": [ "kind", "run_ids" ], "additionalProperties": false } ] } }, "required": [ "op", "source" ], "additionalProperties": false } ] }, "minItems": 1, "maxItems": 100 } }, "required": [ "expected_revision", "edits" ], "additionalProperties": false }, "TestCase": { "type": "object", "properties": { "case_id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z0-9][a-z0-9_-]*$" }, "name": { "type": "string", "minLength": 1, "maxLength": 200 }, "tags": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 64 }, "maxItems": 10, "default": [] }, "mode": { "type": "string", "enum": [ "text", "audio" ], "default": "text" }, "language": { "type": "string", "enum": [ "en", "es" ] }, "caller": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "persona" ] }, "goal": { "type": "string", "minLength": 1, "maxLength": 500 }, "brief": { "type": "string", "minLength": 1, "maxLength": 4000 }, "facts": { "type": "object", "additionalProperties": { "type": "string", "maxLength": 512 }, "default": {} }, "behavior": { "type": "object", "properties": { "verbosity": { "type": "string", "enum": [ "short", "normal", "rambling" ], "default": "short" }, "patience": { "type": "integer", "minimum": 1, "maximum": 10, "default": 3 }, "interrupts": { "type": "string", "enum": [ "never", "sometimes", "often" ], "default": "never" }, "pauses": { "type": "string", "enum": [ "never", "sometimes", "often" ], "default": "never" }, "backchannels": { "type": "string", "enum": [ "never", "sometimes", "often" ], "default": "never" } }, "additionalProperties": false, "default": {} } }, "required": [ "kind", "goal", "brief" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "script" ] }, "steps": { "type": "array", "items": { "type": "object", "properties": { "says": { "type": "string", "minLength": 1, "maxLength": 1000 }, "plays": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "screener_prompt", "voicemail_greeting" ] }, "text": { "type": "string", "minLength": 1, "maxLength": 1000 }, "beep": { "type": "boolean", "default": true } }, "required": [ "kind" ], "additionalProperties": false }, "level_db": { "type": "number", "minimum": -60, "maximum": 0 }, "silence_ms": { "type": "integer", "minimum": 500, "maximum": 600000 }, "pause_before_ms": { "type": "integer", "minimum": 300, "maximum": 5000 }, "interrupt_at_ms": { "type": "integer", "minimum": 0, "maximum": 15000 }, "expect": { "type": "object", "properties": { "lands": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "maxItems": 20 } }, "required": [ "lands" ], "additionalProperties": false } }, "additionalProperties": false }, "minItems": 1, "maxItems": 30 } }, "required": [ "kind", "steps" ], "additionalProperties": false } ] }, "voice": { "type": "object", "properties": { "voice_id": { "type": "string", "minLength": 1, "maxLength": 128 }, "rate": { "type": "number", "minimum": 0.7, "maximum": 1.3, "default": 1 }, "channel": { "type": "string", "enum": [ "pstn", "wideband" ], "default": "pstn" }, "background": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "none", "room", "car", "street", "babble", "music" ] }, "snr_db": { "type": "number", "minimum": 0, "maximum": 40 } }, "required": [ "kind", "snr_db" ], "additionalProperties": false, "default": { "kind": "none", "snr_db": 40 } }, "echo": { "type": "object", "properties": { "level_db": { "type": "number", "minimum": -40, "maximum": -3 }, "delay_ms": { "type": "integer", "minimum": 20, "maximum": 600 } }, "required": [ "level_db", "delay_ms" ], "additionalProperties": false, "nullable": true, "default": null } }, "required": [ "voice_id" ], "additionalProperties": false, "nullable": true, "default": null }, "dynamic_variables": { "type": "object", "additionalProperties": { "type": "string", "maxLength": 4096 }, "default": {} }, "mocks": { "type": "array", "items": { "type": "object", "properties": { "action_id": { "type": "string", "minLength": 1, "maxLength": 128 }, "when": { "type": "array", "items": { "type": "object", "properties": { "path": { "type": "string", "maxLength": 256, "pattern": "^(\\/[^/]*)*$" }, "op": { "type": "string", "enum": [ "equals", "one_of", "contains", "regex", "exists", "absent", "number_range" ] }, "value": {} }, "required": [ "path", "op" ], "additionalProperties": false }, "maxItems": 10, "default": [] }, "respond": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "response" ] }, "status": { "type": "integer", "minimum": 100, "maximum": 599 }, "body": {}, "delay_ms": { "type": "integer", "minimum": 0, "maximum": 60000 } }, "required": [ "kind", "status" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "error" ] }, "error": { "type": "string", "enum": [ "timeout", "network" ] }, "delay_ms": { "type": "integer", "minimum": 0, "maximum": 60000 } }, "required": [ "kind", "error" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "transfer" ] }, "outcome": { "type": "string", "enum": [ "answered", "busy", "no_answer", "failed" ] }, "delay_ms": { "type": "integer", "minimum": 0, "maximum": 60000 } }, "required": [ "kind", "outcome" ], "additionalProperties": false } ] }, "times": { "type": "integer", "exclusiveMinimum": true, "minimum": 0, "maximum": 100 } }, "required": [ "action_id", "respond" ], "additionalProperties": false }, "maxItems": 50, "default": [] }, "max_turns": { "type": "integer", "minimum": 1, "maximum": 30, "default": 12 }, "timeout_ms": { "type": "integer", "minimum": 30000, "maximum": 600000 }, "trials": { "type": "integer", "minimum": 1, "maximum": 30 }, "pass_policy": { "type": "string", "enum": [ "majority", "all" ], "default": "majority" }, "blocking": { "type": "boolean", "default": true }, "assertions": { "type": "array", "items": { "$ref": "#/components/schemas/Assertion" }, "minItems": 1, "maxItems": 20 } }, "required": [ "case_id", "name", "language", "caller", "assertions" ], "additionalProperties": false }, "Assertion": { "anyOf": [ { "type": "object", "properties": { "type": { "type": "string", "enum": [ "node_visited" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "node_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "minItems": 1, "maxItems": 20 }, "match": { "type": "string", "enum": [ "any", "all" ], "default": "any" }, "by_turn": { "type": "integer", "minimum": 0 } }, "required": [ "type", "id", "node_ids" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "node_not_visited" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "node_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "minItems": 1, "maxItems": 20 } }, "required": [ "type", "id", "node_ids" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "path" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "node_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "minItems": 1, "maxItems": 50 }, "order": { "type": "string", "enum": [ "exact", "subsequence" ] } }, "required": [ "type", "id", "node_ids", "order" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "turn_lands" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "step": { "type": "integer", "minimum": 1, "maximum": 30 }, "node_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "maxItems": 20 } }, "required": [ "type", "id", "step", "node_ids" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "final_node" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "node_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128 }, "minItems": 1, "maxItems": 20 } }, "required": [ "type", "id", "node_ids" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "ended" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "reasons": { "type": "array", "items": { "type": "string", "enum": [ "completed", "transferred", "caller_hangup", "max_turns", "timeout", "failed", "cancelled", "inactivity" ] }, "minItems": 1, "maxItems": 8 } }, "required": [ "type", "id", "reasons" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "transfer" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "expected": { "type": "boolean" }, "node_id": { "type": "string", "minLength": 1, "maxLength": 128 } }, "required": [ "type", "id", "expected" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "disposition" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "in": { "type": "array", "items": { "type": "string", "pattern": "^[a-z][a-z0-9_.:-]{0,127}$" }, "minItems": 1, "maxItems": 32 } }, "required": [ "type", "id", "in" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "tool_called" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "action_id": { "type": "string", "minLength": 1, "maxLength": 128 }, "args": { "type": "array", "items": { "type": "object", "properties": { "path": { "type": "string", "maxLength": 256, "pattern": "^(\\/[^/]*)*$" }, "op": { "type": "string", "enum": [ "equals", "one_of", "contains", "regex", "exists", "absent", "number_range" ] }, "value": {} }, "required": [ "path", "op" ], "additionalProperties": false }, "maxItems": 10, "default": [] }, "before_node": { "type": "string", "minLength": 1, "maxLength": 128 }, "times": { "type": "object", "properties": { "min": { "type": "integer", "minimum": 0 }, "max": { "type": "integer", "minimum": 0 } }, "required": [ "min" ], "additionalProperties": false, "default": { "min": 1 } } }, "required": [ "type", "id", "action_id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "tool_not_called" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "action_id": { "type": "string", "minLength": 1, "maxLength": 128 } }, "required": [ "type", "id", "action_id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "variable" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "name": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,63}$" }, "matcher": { "type": "object", "properties": { "op": { "type": "string", "enum": [ "equals", "one_of", "contains", "regex", "exists", "absent", "number_range" ] }, "value": {} }, "required": [ "op" ], "additionalProperties": false } }, "required": [ "type", "id", "name", "matcher" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "agent_says" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "expect": { "type": "string", "enum": [ "never", "at_least_once" ] }, "match": { "anyOf": [ { "type": "object", "properties": { "patterns": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 256 }, "minItems": 1, "maxItems": 20 } }, "required": [ "patterns" ], "additionalProperties": false }, { "type": "object", "properties": { "lexicon": { "type": "string", "enum": [ "debt_disclosure" ] } }, "required": [ "lexicon" ], "additionalProperties": false } ] }, "scope": { "type": "object", "properties": { "until_node": { "type": "string", "minLength": 1, "maxLength": 128 }, "from_turn": { "type": "integer", "minimum": 0 }, "until_step": { "type": "integer", "minimum": 1, "maximum": 30 } }, "additionalProperties": false, "default": {} } }, "required": [ "type", "id", "expect", "match" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "agent_language" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "language": { "type": "string", "enum": [ "en", "es" ] }, "min_share": { "type": "number", "minimum": 0, "maximum": 1 }, "after": { "type": "string", "enum": [ "start", "caller_switch" ] } }, "required": [ "type", "id", "language", "min_share", "after" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "rubric" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "criteria": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z][a-z0-9_]*$" }, "question": { "type": "string", "minLength": 1, "maxLength": 500 }, "scale": { "type": "string", "enum": [ "binary", "1-5" ], "default": "binary" }, "weight": { "type": "integer", "minimum": 1, "maximum": 10, "default": 1 }, "required": { "type": "boolean", "default": false }, "allow_na": { "type": "boolean", "default": false } }, "required": [ "id", "question" ], "additionalProperties": false }, "minItems": 1, "maxItems": 12 }, "threshold": { "type": "number", "minimum": 0, "maximum": 1 } }, "required": [ "type", "id", "criteria", "threshold" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "response_latency" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "p50_ms": { "type": "integer", "exclusiveMinimum": true, "minimum": 0 }, "p95_ms": { "type": "integer", "exclusiveMinimum": true, "minimum": 0 }, "max_ms": { "type": "integer", "exclusiveMinimum": true, "minimum": 0 } }, "required": [ "type", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "greeting_latency" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "max_ms": { "type": "integer", "exclusiveMinimum": true, "minimum": 0 } }, "required": [ "type", "id", "max_ms" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "no_cut_off" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "max": { "type": "integer", "minimum": 0 } }, "required": [ "type", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "barge_in" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "max_stop_ms": { "type": "integer", "minimum": 100, "maximum": 5000, "default": 1000 } }, "required": [ "type", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "no_false_yield" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" } }, "required": [ "type", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "answer_verdict" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "contact": { "type": "array", "items": { "type": "string", "enum": [ "human", "voicemail", "ivr", "unknown" ] }, "minItems": 1, "maxItems": 4 }, "reasons": { "type": "array", "items": { "type": "string", "pattern": "^[a-z][a-z0-9_.:-]{0,127}$" }, "minItems": 1, "maxItems": 10 } }, "required": [ "type", "id", "contact" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "talk_over" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "max_ms": { "type": "integer", "minimum": 0, "maximum": 30000 } }, "required": [ "type", "id", "max_ms" ], "additionalProperties": false } ] }, "TestSuiteView": { "type": "object", "properties": { "owner_id": { "type": "string", "minLength": 1 }, "environment": { "type": "string", "minLength": 1 }, "tenant_id": { "type": "string", "format": "uuid" }, "suite_id": { "type": "string", "format": "uuid" }, "agent_id": { "type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$" }, "name": { "type": "string", "minLength": 1, "maxLength": 200 }, "required": { "type": "boolean" }, "revision": { "type": "integer", "exclusiveMinimum": true, "minimum": 0, "maximum": 2147483647 }, "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, "spec": { "$ref": "#/components/schemas/TestSuiteSpec" }, "created_by": { "$ref": "#/components/schemas/PrincipalRef" }, "created_at": { "type": "string", "format": "date-time" }, "updated_at": { "type": "string", "format": "date-time" }, "archived_at": { "allOf": [ { "type": "string", "format": "date-time" } ], "nullable": true }, "last_run": { "type": "object", "properties": { "test_run_id": { "type": "string", "format": "uuid" }, "version": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "state": { "type": "string", "enum": [ "queued", "running", "completed", "cancelled" ] }, "verdict": { "type": "string", "enum": [ "passed", "failed", "errored" ], "nullable": true }, "created_at": { "type": "string", "format": "date-time" }, "finished_at": { "allOf": [ { "type": "string", "format": "date-time" } ], "nullable": true } }, "required": [ "test_run_id", "version", "state", "verdict", "created_at", "finished_at" ], "additionalProperties": false, "nullable": true }, "flaky_case_ids": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z0-9][a-z0-9_-]*$" }, "maxItems": 100 } }, "required": [ "owner_id", "environment", "tenant_id", "suite_id", "agent_id", "name", "required", "revision", "digest", "spec", "created_by", "created_at", "updated_at", "archived_at", "last_run", "flaky_case_ids" ], "additionalProperties": false }, "TestSuiteSpec": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "simulation" ] }, "cases": { "type": "array", "items": { "$ref": "#/components/schemas/TestCase" }, "minItems": 1, "maxItems": 100 } }, "required": [ "kind", "cases" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "replay" ] }, "source": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "selector" ] }, "window_days": { "type": "integer", "minimum": 1, "maximum": 90 }, "versions": { "type": "array", "items": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "maxItems": 20 }, "contact": { "type": "array", "items": { "type": "string", "enum": [ "human", "voicemail", "ivr", "unknown" ] }, "minItems": 1, "default": [ "human" ] }, "dispositions": { "type": "array", "items": { "type": "string", "pattern": "^[a-z][a-z0-9_.:-]{0,127}$" }, "maxItems": 32 }, "min_caller_turns": { "type": "integer", "minimum": 1, "default": 1 }, "sample": { "type": "string", "enum": [ "latest", "stratified_by_disposition" ], "default": "latest" }, "limit": { "type": "integer", "minimum": 1, "maximum": 200 } }, "required": [ "kind", "window_days", "limit" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "runs" ] }, "run_ids": { "type": "array", "items": { "type": "string", "format": "uuid" }, "minItems": 1, "maxItems": 200 } }, "required": [ "kind", "run_ids" ], "additionalProperties": false } ] }, "trials": { "type": "integer", "minimum": 1, "maximum": 3, "default": 1 }, "population": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "type": { "type": "string", "enum": [ "regression" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "metric": { "type": "string", "enum": [ "outcome_agreement", "edge_agreement" ] }, "tolerance_per_30": { "type": "integer", "minimum": 0, "maximum": 30, "default": 1 } }, "required": [ "type", "id", "metric" ], "additionalProperties": false }, { "type": "object", "properties": { "type": { "type": "string", "enum": [ "judged_divergence" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, "max_judged_worse": { "type": "integer", "minimum": 0 } }, "required": [ "type", "id", "max_judged_worse" ], "additionalProperties": false } ] }, "minItems": 1, "maxItems": 5 }, "every_case": { "type": "array", "items": { "$ref": "#/components/schemas/Assertion" }, "maxItems": 10, "default": [] } }, "required": [ "kind", "source", "population" ], "additionalProperties": false } ] }, "PrincipalRef": { "anyOf": [ { "type": "object", "properties": { "owner_id": { "type": "string", "minLength": 1 }, "environment": { "type": "string", "minLength": 1 }, "kind": { "type": "string", "enum": [ "user" ] }, "id": { "type": "string", "format": "uuid" } }, "required": [ "owner_id", "environment", "kind", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "owner_id": { "type": "string", "minLength": 1 }, "environment": { "type": "string", "minLength": 1 }, "kind": { "type": "string", "enum": [ "api_key" ] }, "id": { "type": "string", "format": "uuid" } }, "required": [ "owner_id", "environment", "kind", "id" ], "additionalProperties": false }, { "type": "object", "properties": { "owner_id": { "type": "string", "minLength": 1 }, "environment": { "type": "string", "minLength": 1 }, "kind": { "type": "string", "enum": [ "system" ] }, "id": { "type": "string", "minLength": 1, "maxLength": 256 } }, "required": [ "owner_id", "environment", "kind", "id" ], "additionalProperties": false } ] }, "ApiError": { "type": "object", "properties": { "error": { "type": "object", "properties": { "code": { "type": "string", "enum": [ "unauthenticated", "forbidden", "not_found", "conflict", "invalid_request", "idempotency_conflict", "policy_denied", "capability_unavailable", "provider_unavailable", "knowledge_unavailable", "publication_invalid", "run_not_active", "rate_limited", "outcome_unknown", "internal" ] }, "message": { "type": "string" }, "retryable": { "type": "boolean" }, "outcome": { "type": "string", "enum": [ "none", "unknown" ] }, "details": { "type": "object", "additionalProperties": {} } }, "required": [ "code", "message", "retryable", "outcome" ], "additionalProperties": false } }, "required": [ "error" ], "additionalProperties": false } } } } ``` ## Errors and recovery {#errors} See [API error handling](https://docs.cumuluslabs.io/api#errors) and [Troubleshooting](https://docs.cumuluslabs.io/troubleshooting). ## Related guides {#related-guides} - [Test an agent](https://docs.cumuluslabs.io/testing) - [Build and test recipes](https://docs.cumuluslabs.io/recipes)