{ "openapi": "3.1.0", "info": { "title": "Pensieve API", "version": "2026-11-08", "summary": "Find the post-training that gets the best outcome from your data, and run the model it makes.", "description": "Pensieve runs experiments on your data. For each decision your software makes, its agents look for\nthe post-training that gets the best outcome. Each experiment trains the current recipe (A) and one\nchange (B) on the same three seeds, and scores both on your held-out test, which no model trains on.\nA change is kept only if B wins on every seed and on a last check against cases neither has seen.\nKept changes add up to a recipe; a trained recipe is a version. A version ships once it beats the live\nversion and the best frontier model on that test, runs in shadow, and is signed off by your expert.\nEach post-training method is a plain markdown file the agents follow: Pensieve's own, or one you write.\n\nYour software calls a model with a case and gets back one of its fixed answers, a confidence and the\nversion that answered. What really happened next goes back in, and feeds the next experiments.\n\n**Keys.** Send `Authorization: Bearer `. A key that starts `pk_test_` reaches a sandbox\nworkspace, where models answer from sample data; `pk_live_` reaches your live workspace.\n`pk_test_sandbox` is a public key everyone shares. Any other `pk_test_` key you make up gets a sandbox\nworkspace of its own, with the same sample data.\n\n**The sandbox.** It keeps your changes in memory for about an hour, and may reset sooner. Waits are\nskipped: shadow runs and batches finish as soon as they start, so you can try a whole release in a\nminute. Nothing trains there: experiments you start stay queued, and weight downloads answer 501. It\nrecords events at `/v1/events` but does not send webhooks yet.\n\n**Versions.** Pin the API version with `Pensieve-Version: 2026-11-08`. Without the header you get\nthe version your workspace was created on.\n\n**Retries.** Every POST takes an `Idempotency-Key` header. Sending the same key again within 24 hours\nreturns the first response instead of doing the work twice.\n\n**Errors.** Every error has the same body: `{\"error\": {\"type\", \"message\", \"param\", \"request_id\"}}`.\nA `not_available` error (501) means the operation is part of the API but not served by the sandbox\nyet.\n\n**Enums.** Enum values in responses may grow: a new status, answer kind or event type can appear\nwithout a new API version. Treat a value you don't know as \"other\" rather than failing.\n\n**Lists.** Lists return `{\"object\": \"list\", \"data\": [...], \"has_more\", \"next_cursor\"}`; pass\n`cursor` to get the next page.\n\n**Webhooks.** Each delivery carries `Pensieve-Signature: t=,v1=`, where `v1` is\nthe HMAC-SHA256 of `.` with the endpoint's secret. Reject deliveries more than five\nminutes old. The SDKs check both for you.\n", "contact": { "name": "Pensieve", "email": "krish@pensievelabs.co" } }, "servers": [ { "url": "https://api.pensievelabs.co", "description": "Sandbox and live; the key decides which workspace you reach." }, { "url": "http://localhost:4410", "description": "The sandbox, run locally from apps/api." } ], "security": [ { "bearer": [] } ], "tags": [ { "name": "Models", "description": "Your expert models, their answers and how calls are routed." }, { "name": "Decisions", "description": "Calling a model, one case or many." }, { "name": "Feedback", "description": "Calls, what really happened, experts' answers, and calls made elsewhere." }, { "name": "Test", "description": "The held-out test every version and frontier model is scored on." }, { "name": "Experiments", "description": "Pensieve's agents look for the post-training that gets the best outcome from your data. Each experiment\ntrains the current recipe (A) and one change (B) on the same three seeds and scores both on your held-out\ntest. A change is kept only if B wins on every seed and on a last check against cases neither has seen.\n" }, { "name": "Methods", "description": "The post-training methods experiments can use, each a plain markdown file the agents read: when to try\nit, what it needs, its steps, the settings an experiment can change, and what to check. Pensieve ships\na few; write your own, or copy one and change it.\n" }, { "name": "Releases", "description": "Versions, shadow runs, sign-off, rollout and rollback." }, { "name": "Review", "description": "The queue of cases and releases waiting for your experts." }, { "name": "Tasks", "description": "Giving Pensieve a new decision to build a model for." }, { "name": "Frontier", "description": "Re-tests after every frontier and general model release." }, { "name": "Workspace", "description": "Data sources, keys, people, usage and webhooks." } ], "paths": { "/v1/workspace": { "get": { "operationId": "retrieveWorkspace", "tags": [ "Workspace" ], "summary": "The workspace this key reaches", "x-sdk": { "resource": "workspace", "method": "retrieve" }, "responses": { "200": { "description": "The workspace.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models": { "get": { "operationId": "listModels", "tags": [ "Models" ], "summary": "List models", "x-sdk": { "resource": "models", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" }, { "name": "status", "in": "query", "schema": { "$ref": "#/components/schemas/ModelStatus" } } ], "responses": { "200": { "description": "A page of models.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "retrieveModel", "tags": [ "Models" ], "summary": "Retrieve a model", "x-sdk": { "resource": "models", "method": "retrieve" }, "responses": { "200": { "description": "The model.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "patch": { "operationId": "updateModel", "tags": [ "Models" ], "summary": "Rename a model, pause its experiments, or set its budget", "x-sdk": { "resource": "models", "method": "update" }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelUpdate" }, "example": { "experimenting": true, "monthly_budget_usd": 2000 } } } }, "responses": { "200": { "description": "The model.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/decide": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "post": { "operationId": "decide", "tags": [ "Decisions" ], "summary": "Decide one case", "description": "Sends one case to the model's live version (or the version you pin) and returns one of its\nanswers. When `routed` is `fallback`, the answer is still returned, but your code, or the SDK's\n`capture`, should answer instead. A model with no live version answers only with `version`\npinned; otherwise the call is refused with `conflict`.\n", "x-sdk": { "resource": "models", "method": "decide" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DecideRequest" }, "example": { "input": { "task": "rust-2291", "grade": "pass", "log": "Grader: PASS · 14 of 14 tests passed. Test change: expected page size 20 → 21" }, "key": "rust-2291" } } } }, "responses": { "200": { "description": "The decision.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Decision" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/routing": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "patch": { "operationId": "updateRouting", "tags": [ "Models" ], "summary": "Change how calls are routed", "description": "`observe` records calls only: your own code keeps answering, through `capture`. `model` sends\ncalls to the expert model and falls back under `threshold`. `split` sends `split_percent` of\ncalls to the model and the rest to your code.\n", "x-sdk": { "resource": "models.routing", "method": "update" }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RoutingUpdate" }, "example": { "mode": "model", "threshold": 0.8 } } } }, "responses": { "200": { "description": "The routing now in force.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Routing" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/versions": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "listVersions", "tags": [ "Releases" ], "summary": "List a model's versions", "x-sdk": { "resource": "models.versions", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "Versions, newest first.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/VersionList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/versions/{version}": { "parameters": [ { "$ref": "#/components/parameters/Model" }, { "$ref": "#/components/parameters/Version" } ], "get": { "operationId": "retrieveVersion", "tags": [ "Releases" ], "summary": "Retrieve a version", "x-sdk": { "resource": "models.versions", "method": "retrieve" }, "responses": { "200": { "description": "The version.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelVersion" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/versions/{version}/weights": { "parameters": [ { "$ref": "#/components/parameters/Model" }, { "$ref": "#/components/parameters/Version" } ], "get": { "operationId": "retrieveWeights", "tags": [ "Releases" ], "summary": "Download links for a paid version's weights", "description": "You keep the weights of every version you have paid for. Links last one hour.", "x-sdk": { "resource": "models.versions", "method": "weights" }, "responses": { "200": { "description": "Signed download links.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Weights" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/batches": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "post": { "operationId": "createBatch", "tags": [ "Decisions" ], "summary": "Decide many cases", "description": "Up to 10,000 cases. Poll the batch, or listen for `batch.completed`.", "x-sdk": { "resource": "batches", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchCreate" }, "example": { "cases": [ { "input": "Grader: FAIL · TestRetryTimeout timed out after 30s on runner ci-7", "key": "go-118" }, { "input": "Grader: PASS · 22 of 22 tests passed, existing tests unchanged", "key": "ts-402" } ] } } } }, "responses": { "200": { "description": "The batch.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Batch" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/batches/{batch_id}": { "parameters": [ { "name": "batch_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "bat_5d2e81f0" } ], "get": { "operationId": "retrieveBatch", "tags": [ "Decisions" ], "summary": "Retrieve a batch", "x-sdk": { "resource": "batches", "method": "retrieve" }, "responses": { "200": { "description": "The batch, with its decisions once done.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Batch" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/calls": { "get": { "operationId": "listCalls", "tags": [ "Feedback" ], "summary": "List calls", "description": "Decisions made by Pensieve and calls recorded with `capture`, newest first.", "x-sdk": { "resource": "calls", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" }, { "name": "model", "in": "query", "schema": { "type": "string" }, "example": "grade-audit" }, { "name": "key", "in": "query", "description": "Your own reference for the case.", "schema": { "type": "string" } }, { "name": "source", "in": "query", "schema": { "$ref": "#/components/schemas/CallSource" } } ], "responses": { "200": { "description": "A page of calls.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CallList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/calls/{call_id}": { "parameters": [ { "$ref": "#/components/parameters/CallId" } ], "get": { "operationId": "retrieveCall", "tags": [ "Feedback" ], "summary": "Retrieve a call", "x-sdk": { "resource": "calls", "method": "retrieve" }, "responses": { "200": { "description": "The call, its outcome and any correction.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Call" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/calls/{call_id}/outcome": { "parameters": [ { "$ref": "#/components/parameters/CallId" } ], "post": { "operationId": "sendOutcome", "tags": [ "Feedback" ], "summary": "Send what really happened", "description": "Outcomes check every version against reality and catch drift. A later outcome replaces an earlier one.\nAn outcome can arrive before its call: it is kept, and matched when a call with that id is recorded.\n", "x-sdk": { "resource": "outcomes", "method": "send" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OutcomeCreate" }, "example": { "outcome": "lab_rejected", "note": "Lab found the edited test." } } } }, "responses": { "200": { "description": "The outcome as recorded.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Outcome" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/calls/{call_id}/correction": { "parameters": [ { "$ref": "#/components/parameters/CallId" } ], "post": { "operationId": "sendCorrection", "tags": [ "Feedback" ], "summary": "Send an expert's answer", "description": "An expert's answer for the case. It becomes training data, and may become a test case.", "x-sdk": { "resource": "corrections", "method": "send" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CorrectionCreate" }, "example": { "answer": "false_pass", "by": "maya@example.com", "note": "The patch edits its own test." } } } }, "responses": { "200": { "description": "The correction as recorded.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Correction" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/outcomes": { "post": { "operationId": "sendOutcomes", "tags": [ "Feedback" ], "summary": "Send many outcomes", "description": "Each item names its call by `call_id`, or by `model` and your own `key`. An item whose call isn't\nrecorded yet is kept, and matched when a call with that id, or that model and key, arrives.\n", "x-sdk": { "resource": "outcomes", "method": "send_many" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OutcomeBulk" }, "example": { "items": [ { "model": "grade-audit", "key": "rust-2291", "outcome": "lab_rejected" } ] } } } }, "responses": { "200": { "description": "How many were recorded.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Receipt" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/captures": { "post": { "operationId": "createCaptures", "tags": [ "Feedback" ], "summary": "Record calls answered elsewhere", "description": "Calls your own code answered, usually a frontier model, recorded so Pensieve can build the\ntest and training data from real traffic. The SDK's `capture` sends these for you.\n", "x-sdk": { "resource": "captures", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CaptureCreate" }, "example": { "model": "grade-audit", "calls": [ { "input": { "task": "rust-2291", "grade": "pass" }, "answer": "grade_holds", "source": "frontier", "provider_model": "frontier-large", "latency_ms": 2140, "key": "rust-2291" } ] } } } }, "responses": { "200": { "description": "The ids given to the recorded calls.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Receipt" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/test": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "retrieveTest", "tags": [ "Test" ], "summary": "The model's test and every score on it", "x-sdk": { "resource": "tests", "method": "retrieve" }, "responses": { "200": { "description": "The test.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Test" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/test/cases": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "listTestCases", "tags": [ "Test" ], "summary": "List test cases", "x-sdk": { "resource": "tests.cases", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "A page of test cases.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TestCaseList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "addTestCases", "tags": [ "Test" ], "summary": "Add test cases", "description": "New cases are held out from training. Every version and frontier model is scored on them again.", "x-sdk": { "resource": "tests.cases", "method": "add" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TestCasesAdd" }, "example": { "cases": [ { "input": "Grader: PASS · patch deletes the failing assertion", "answer": "false_pass", "by": "maya@example.com" } ] } } } }, "responses": { "200": { "description": "The ids of the added cases.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Receipt" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/experiments": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "listExperiments", "tags": [ "Experiments" ], "summary": "List experiments", "x-sdk": { "resource": "experiments", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" }, { "name": "status", "in": "query", "schema": { "$ref": "#/components/schemas/ExperimentStatus" } } ], "responses": { "200": { "description": "Experiments, newest first.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "startExperiment", "tags": [ "Experiments" ], "summary": "Suggest an experiment", "description": "An idea for the agents to test, such as a data source to weight differently or a reward to try. They\nwrite the hypothesis, pick the method if you don't, build B's data, and run A and B on three seeds.\nRefused with `conflict` while the model's experiments are paused, or once this month's budget is spent.\n", "x-sdk": { "resource": "experiments", "method": "start" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentCreate" }, "example": { "idea": "When two reviewers disagree, train on the senior reviewer's grade", "method": "fine-tuning", "max_cost_usd": 80 } } } }, "responses": { "200": { "description": "The experiment, queued or running.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Experiment" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/experiments/{experiment_id}": { "parameters": [ { "$ref": "#/components/parameters/ExperimentId" } ], "get": { "operationId": "retrieveExperiment", "tags": [ "Experiments" ], "summary": "Retrieve an experiment", "x-sdk": { "resource": "experiments", "method": "retrieve" }, "responses": { "200": { "description": "The experiment.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Experiment" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/experiments/{experiment_id}/runs": { "parameters": [ { "$ref": "#/components/parameters/ExperimentId" } ], "get": { "operationId": "listExperimentRuns", "tags": [ "Experiments" ], "summary": "List an experiment's runs", "description": "Its prep step, A and B on every seed, and the last check, with each run's progress and score.", "x-sdk": { "resource": "experiments.runs", "method": "list" }, "responses": { "200": { "description": "The runs, in order.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentRunList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/experiments/{experiment_id}/stop": { "parameters": [ { "$ref": "#/components/parameters/ExperimentId" } ], "post": { "operationId": "stopExperiment", "tags": [ "Experiments" ], "summary": "Stop an experiment", "description": "Runs that finished keep their results.", "x-sdk": { "resource": "experiments", "method": "stop" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "responses": { "200": { "description": "The stopped experiment.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Experiment" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/methods": { "get": { "operationId": "listMethods", "tags": [ "Methods" ], "summary": "List methods", "description": "The built-in methods and your own, in one list.", "x-sdk": { "resource": "methods", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "A page of methods.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MethodList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "createMethod", "tags": [ "Methods" ], "summary": "Add a method", "description": "A post-training method of your own, as a markdown file. From then on the agents can choose it for an\nexperiment, and you can name it when you suggest one. A slug is used only once: the slug of a removed\nmethod can't be taken again.\n", "x-sdk": { "resource": "methods", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MethodCreate" }, "example": { "slug": "curriculum", "markdown": "# Curriculum fine-tuning\n\nFine-tune on easy cases first, then hard ones.\n" } } } }, "responses": { "200": { "description": "The method.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Method" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/methods/{method}": { "parameters": [ { "$ref": "#/components/parameters/Method" } ], "get": { "operationId": "retrieveMethod", "tags": [ "Methods" ], "summary": "Retrieve a method", "description": "A removed method answers `not_found` unless you pass `version`; every version it had stays readable.", "x-sdk": { "resource": "methods", "method": "retrieve" }, "parameters": [ { "name": "version", "in": "query", "schema": { "type": "integer", "minimum": 1 }, "description": "An earlier version of the file, such as the one an experiment ran. Default: the latest. Needed for a removed method." } ], "responses": { "200": { "description": "The method.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Method" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "patch": { "operationId": "updateMethod", "tags": [ "Methods" ], "summary": "Change a method", "description": "Replaces the file and raises its version by one. Experiments already designed keep the version they\nstarted with. Built-in methods can't be changed: add a copy under a new slug.\n", "x-sdk": { "resource": "methods", "method": "update" }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MethodUpdate" }, "example": { "markdown": "# Curriculum fine-tuning\n\nFine-tune on easy cases first, then hard ones, in three stages.\n" } } } }, "responses": { "200": { "description": "The method, at its new version.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Method" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "delete": { "operationId": "removeMethod", "tags": [ "Methods" ], "summary": "Remove a method", "description": "It leaves the list, and the agents stop choosing it. Experiments that used it keep it on record, and\neach version stays readable with `version`, so a result still leads back to the text the agents\nfollowed. Its slug can't be used again. A method a queued or running experiment uses, or a built-in\none, can't be removed.\n", "x-sdk": { "resource": "methods", "method": "remove" }, "responses": { "200": { "description": "The removed method.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Deleted" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/releases": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "get": { "operationId": "listReleases", "tags": [ "Releases" ], "summary": "List releases", "x-sdk": { "resource": "releases", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "Releases, newest first.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReleaseList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "startShadowRun", "tags": [ "Releases" ], "summary": "Start a shadow run", "description": "A version that beats the live version and the best frontier model on your test runs beside\nlive traffic for `shadow_days`; nothing it decides reaches your users. Refused with `conflict`\nwhile a gate before the shadow run is not passed.\n", "x-sdk": { "resource": "releases", "method": "shadow" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReleaseCreate" }, "example": { "version": "v8", "shadow_days": 7 } } } }, "responses": { "200": { "description": "The release in its shadow run.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/releases/{release_id}": { "parameters": [ { "$ref": "#/components/parameters/ReleaseId" } ], "get": { "operationId": "retrieveRelease", "tags": [ "Releases" ], "summary": "Retrieve a release", "x-sdk": { "resource": "releases", "method": "retrieve" }, "responses": { "200": { "description": "The release and its gates.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/releases/{release_id}/signoff": { "parameters": [ { "$ref": "#/components/parameters/ReleaseId" } ], "post": { "operationId": "signOffRelease", "tags": [ "Releases" ], "summary": "Sign off a release", "description": "Only the people named in the workspace's release rules can sign off. Refused with `conflict` before the shadow run ends.", "x-sdk": { "resource": "releases", "method": "signoff" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SignOff" }, "example": { "by": "maya@example.com", "note": "Matches our reviewers on the Rust tasks." } } } }, "responses": { "200": { "description": "The signed-off release.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/releases/{release_id}/split": { "parameters": [ { "$ref": "#/components/parameters/ReleaseId" } ], "post": { "operationId": "splitRelease", "tags": [ "Releases" ], "summary": "Send part of the traffic to a signed-off version", "x-sdk": { "resource": "releases", "method": "split" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SplitRequest" }, "example": { "percent": 10 } } } }, "responses": { "200": { "description": "The release with its split.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/releases/{release_id}/promote": { "parameters": [ { "$ref": "#/components/parameters/ReleaseId" } ], "post": { "operationId": "promoteRelease", "tags": [ "Releases" ], "summary": "Make a signed-off version live", "description": "Succeeds only if the live version is still `expect`, so two people cannot promote over each other.", "x-sdk": { "resource": "releases", "method": "promote" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PromoteRequest" }, "example": { "expect": "v7" } } } }, "responses": { "200": { "description": "The release, now live.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/releases/{release_id}/stop": { "parameters": [ { "$ref": "#/components/parameters/ReleaseId" } ], "post": { "operationId": "stopRelease", "tags": [ "Releases" ], "summary": "Stop a shadow run or a split", "x-sdk": { "resource": "releases", "method": "stop" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "responses": { "200": { "description": "The stopped release.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Release" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/models/{model}/rollback": { "parameters": [ { "$ref": "#/components/parameters/Model" } ], "post": { "operationId": "rollback", "tags": [ "Releases" ], "summary": "Go back to the previous version", "description": "The previous version stays warm, so traffic moves back in seconds.", "x-sdk": { "resource": "releases", "method": "rollback" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RollbackRequest" }, "example": { "expect": "v7", "reason": "Agreement dropped after a harness change." } } } }, "responses": { "200": { "description": "The model, with the version now live.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/review": { "get": { "operationId": "listReviewCases", "tags": [ "Review" ], "summary": "List the review queue", "x-sdk": { "resource": "review", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" }, { "name": "model", "in": "query", "schema": { "type": "string" } }, { "name": "state", "in": "query", "schema": { "type": "string", "enum": [ "open", "answered" ] } }, { "name": "kind", "in": "query", "schema": { "$ref": "#/components/schemas/ReviewKind" } } ], "responses": { "200": { "description": "A page of review cases.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReviewCaseList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/review/{case_id}": { "parameters": [ { "$ref": "#/components/parameters/CaseId" } ], "get": { "operationId": "retrieveReviewCase", "tags": [ "Review" ], "summary": "Retrieve a review case", "x-sdk": { "resource": "review", "method": "retrieve" }, "responses": { "200": { "description": "The review case.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReviewCase" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/review/{case_id}/answer": { "parameters": [ { "$ref": "#/components/parameters/CaseId" } ], "post": { "operationId": "answerReviewCase", "tags": [ "Review" ], "summary": "Answer a review case", "x-sdk": { "resource": "review", "method": "answer" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReviewAnswer" }, "example": { "answer": "false_pass", "by": "maya@example.com", "add_to_test": true } } } }, "responses": { "200": { "description": "The answered case.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReviewCase" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/tasks": { "post": { "operationId": "createTask", "tags": [ "Tasks" ], "summary": "Assign a task", "description": "Describe a decision your software makes. Pensieve's agents run experiments on your data to find the\npost-training for it, starting with a first recipe that is queued at once. Kept changes add up to a\nrecipe; a trained recipe is a version. A version ships once it beats the live version (if there is\none) and the best frontier model on your test, runs in shadow, and is signed off by your expert.\n", "x-sdk": { "resource": "tasks", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TaskCreate" }, "example": { "name": "Flaky test triage", "decision": "Decide whether a failing CI test is a real bug or a flaky test, before it pages an engineer.", "answers": [ { "value": "real_bug" }, { "value": "flaky_test" } ], "test": { "held_out_cases": 2000, "metric": "Right call on each failing test" }, "runs_in": "your_vpc", "monthly_budget_usd": 1500 } } } }, "responses": { "200": { "description": "The new model, building.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/frontier": { "get": { "operationId": "listFrontierRetests", "tags": [ "Frontier" ], "summary": "List re-tests after frontier releases", "x-sdk": { "resource": "frontier", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "Re-tests, newest first.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RetestList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/sources": { "get": { "operationId": "listSources", "tags": [ "Workspace" ], "summary": "List data sources", "x-sdk": { "resource": "sources", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" } ], "responses": { "200": { "description": "A page of sources.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SourceList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "connectSource", "tags": [ "Workspace" ], "summary": "Connect a data source", "description": "Pensieve's reader runs where your data is; credentials are set in the app, never sent here.", "x-sdk": { "resource": "sources", "method": "connect" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SourceCreate" }, "example": { "name": "CI runs", "kind": "webhook", "models": [ "grade-audit" ] } } } }, "responses": { "200": { "description": "The source, connecting.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Source" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/sources/{source_id}": { "parameters": [ { "name": "source_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "src_agent_trials" } ], "get": { "operationId": "retrieveSource", "tags": [ "Workspace" ], "summary": "Retrieve a source", "x-sdk": { "resource": "sources", "method": "retrieve" }, "responses": { "200": { "description": "The source.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Source" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "patch": { "operationId": "updateSource", "tags": [ "Workspace" ], "summary": "Pause or resume a source", "x-sdk": { "resource": "sources", "method": "update" }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SourceUpdate" }, "example": { "state": "paused" } } } }, "responses": { "200": { "description": "The source.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Source" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "delete": { "operationId": "removeSource", "tags": [ "Workspace" ], "summary": "Disconnect a source", "description": "Data already read stays in the models trained on it.", "x-sdk": { "resource": "sources", "method": "remove" }, "responses": { "200": { "description": "The removed source.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Deleted" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/keys": { "get": { "operationId": "listKeys", "tags": [ "Workspace" ], "summary": "List API keys", "x-sdk": { "resource": "keys", "method": "list" }, "responses": { "200": { "description": "The keys, without their secrets.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKeyList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "createKey", "tags": [ "Workspace" ], "summary": "Create an API key", "description": "The secret is in this response only.", "x-sdk": { "resource": "keys", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKeyCreate" }, "example": { "name": "Staging services" } } } }, "responses": { "200": { "description": "The key and its secret.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKey" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/keys/{key_id}": { "parameters": [ { "name": "key_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "key_8c1f20" } ], "delete": { "operationId": "revokeKey", "tags": [ "Workspace" ], "summary": "Revoke an API key", "x-sdk": { "resource": "keys", "method": "revoke" }, "responses": { "200": { "description": "The revoked key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKey" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/members": { "get": { "operationId": "listMembers", "tags": [ "Workspace" ], "summary": "List people", "x-sdk": { "resource": "members", "method": "list" }, "responses": { "200": { "description": "The workspace's people.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MemberList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "inviteMember", "tags": [ "Workspace" ], "summary": "Invite someone", "x-sdk": { "resource": "members", "method": "invite" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MemberInvite" }, "example": { "email": "dana@example.com", "role": "expert" } } } }, "responses": { "200": { "description": "The invited member.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Member" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/members/{member_id}": { "parameters": [ { "name": "member_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "mem_maya" } ], "patch": { "operationId": "updateMember", "tags": [ "Workspace" ], "summary": "Change someone's role", "x-sdk": { "resource": "members", "method": "update" }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MemberUpdate" }, "example": { "role": "admin" } } } }, "responses": { "200": { "description": "The member.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Member" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "delete": { "operationId": "removeMember", "tags": [ "Workspace" ], "summary": "Remove someone, or cancel an invite", "x-sdk": { "resource": "members", "method": "remove" }, "responses": { "200": { "description": "The removed member.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Deleted" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/usage": { "get": { "operationId": "retrieveUsage", "tags": [ "Workspace" ], "summary": "Calls and compute for a month", "x-sdk": { "resource": "usage", "method": "retrieve" }, "parameters": [ { "name": "month", "in": "query", "description": "The month as YYYY-MM; this month if left out.", "schema": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}$" }, "example": "2026-11" } ], "responses": { "200": { "description": "Usage for the month.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Usage" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/webhooks": { "get": { "operationId": "listWebhooks", "tags": [ "Workspace" ], "summary": "List webhook endpoints", "x-sdk": { "resource": "webhooks", "method": "list" }, "responses": { "200": { "description": "The endpoints.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookList" } } } }, "default": { "$ref": "#/components/responses/Error" } } }, "post": { "operationId": "createWebhook", "tags": [ "Workspace" ], "summary": "Add a webhook endpoint", "description": "Events are signed with the endpoint's secret in `Pensieve-Signature`; the SDKs check it for you.", "x-sdk": { "resource": "webhooks", "method": "create" }, "parameters": [ { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WebhookCreate" }, "example": { "url": "https://example.com/hooks/pensieve", "events": [ "release.ready_for_signoff", "frontier.retested" ] } } } }, "responses": { "200": { "description": "The endpoint and its signing secret.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Webhook" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/webhooks/{webhook_id}": { "parameters": [ { "name": "webhook_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "whe_3b7c11" } ], "delete": { "operationId": "removeWebhook", "tags": [ "Workspace" ], "summary": "Remove a webhook endpoint", "x-sdk": { "resource": "webhooks", "method": "remove" }, "responses": { "200": { "description": "The removed endpoint.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Deleted" } } } }, "default": { "$ref": "#/components/responses/Error" } } } }, "/v1/events": { "get": { "operationId": "listEvents", "tags": [ "Workspace" ], "summary": "List recent events", "description": "The same events webhooks deliver, kept 30 days, for polling or catching up.", "x-sdk": { "resource": "events", "method": "list" }, "parameters": [ { "$ref": "#/components/parameters/Limit" }, { "$ref": "#/components/parameters/Cursor" }, { "name": "type", "in": "query", "schema": { "$ref": "#/components/schemas/EventType" } } ], "responses": { "200": { "description": "Events, newest first.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EventList" } } } }, "default": { "$ref": "#/components/responses/Error" } } } } }, "webhooks": { "release.ready_for_signoff": { "post": { "summary": "A release finished its shadow run and waits for sign-off", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "release.promoted": { "post": { "summary": "A version went live", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "release.rolled_back": { "post": { "summary": "A model went back to its previous version", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "frontier.retested": { "post": { "summary": "A frontier or general model release was scored on every task's test", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "model.lead_narrowed": { "post": { "summary": "A re-test narrowed a model's lead; agents have started experiments", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "experiment.ended": { "post": { "summary": "An experiment was kept, dropped or stopped", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "review.case_added": { "post": { "summary": "A case or a release joined the review queue", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } }, "batch.completed": { "post": { "summary": "A batch finished", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Event" } } } }, "responses": { "200": { "description": "Return any 2xx to acknowledge." } } } } }, "components": { "securitySchemes": { "bearer": { "type": "http", "scheme": "bearer", "description": "An API key from the app's Settings. `pk_test_` keys reach the sandbox." } }, "parameters": { "Model": { "name": "model", "in": "path", "required": true, "description": "The model's slug, such as `grade-audit`, or its `mdl_` id.", "schema": { "type": "string" }, "example": "grade-audit" }, "Version": { "name": "version", "in": "path", "required": true, "schema": { "type": "string" }, "example": "v7" }, "CallId": { "name": "call_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "cl_seed_grade_01" }, "ExperimentId": { "name": "experiment_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "exp_0420" }, "Method": { "name": "method", "in": "path", "required": true, "schema": { "$ref": "#/components/schemas/MethodSlug" }, "example": "fine-tuning" }, "ReleaseId": { "name": "release_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "rel_grade_audit_v8" }, "CaseId": { "name": "case_id", "in": "path", "required": true, "schema": { "type": "string" }, "example": "rev_grade_01" }, "Limit": { "name": "limit", "in": "query", "schema": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 } }, "Cursor": { "name": "cursor", "in": "query", "description": "The `next_cursor` of the page before.", "schema": { "type": "string" } }, "IdempotencyKey": { "name": "Idempotency-Key", "in": "header", "description": "Any unique string. Sending it again within 24 hours returns the first response.", "schema": { "type": "string", "maxLength": 255 } } }, "responses": { "Error": { "description": "Something went wrong; `error.type` says what.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "schemas": { "Error": { "type": "object", "required": [ "error" ], "properties": { "error": { "$ref": "#/components/schemas/ErrorDetail" } } }, "ErrorDetail": { "type": "object", "required": [ "type", "message", "request_id" ], "properties": { "type": { "type": "string", "enum": [ "invalid_request", "authentication", "permission", "not_found", "conflict", "rate_limit", "not_available", "server" ] }, "message": { "type": "string" }, "param": { "type": [ "string", "null" ], "description": "The request field the error is about." }, "request_id": { "type": "string" } } }, "Timestamp": { "type": "string", "format": "date-time" }, "ListMeta": { "type": "object", "required": [ "object", "has_more", "next_cursor" ], "properties": { "object": { "const": "list" }, "has_more": { "type": "boolean" }, "next_cursor": { "type": [ "string", "null" ] } } }, "Receipt": { "type": "object", "required": [ "object", "received", "ids" ], "properties": { "object": { "const": "receipt" }, "received": { "type": "integer", "description": "How many items were recorded." }, "ids": { "type": "array", "description": "For each item in order, the call or test case it was recorded as (for a capture with `decision_id`,\nthat decision's call). For an outcome, the call it was matched to, or null while it waits for its\ncall: it is matched when a call with that id, or that model and key, is recorded.\n", "items": { "type": [ "string", "null" ] } } } }, "Deleted": { "type": "object", "required": [ "id", "object", "deleted" ], "properties": { "id": { "type": "string" }, "object": { "type": "string" }, "deleted": { "const": true } } }, "Workspace": { "type": "object", "required": [ "id", "object", "name", "mode", "region", "created_at" ], "properties": { "id": { "type": "string", "example": "ws_demo" }, "object": { "const": "workspace" }, "name": { "type": "string" }, "mode": { "type": "string", "enum": [ "sandbox", "live" ] }, "region": { "type": "string", "example": "Your VPC · us-east-1" }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "ModelStatus": { "type": "string", "enum": [ "building", "shadow", "live", "paused", "archived" ], "description": "`building`: no version beats the frontier yet. `shadow`: a version runs beside live traffic. `live`: a\nversion takes traffic. `paused`: taken off traffic in the app, so your code answers every call until\nit is resumed. `archived`: retired in the app; it answers no calls, and lists leave it out unless you\nask for `status=archived`.\n" }, "Answer": { "type": "object", "required": [ "value", "label" ], "properties": { "value": { "type": "string", "description": "What your code receives.", "example": "false_pass" }, "label": { "type": "string", "example": "False pass" }, "description": { "type": [ "string", "null" ] } } }, "Lead": { "type": "object", "description": "The live version (or best candidate) against the best frontier model, on your test.", "required": [ "model_score", "frontier_score", "points", "frontier_model", "tested_at" ], "properties": { "model_score": { "type": "number" }, "frontier_score": { "type": "number" }, "points": { "type": "number", "description": "The lead in points out of 100; negative when behind." }, "frontier_model": { "type": "string", "description": "The best frontier model at the last re-test." }, "tested_at": { "$ref": "#/components/schemas/Timestamp" } } }, "Routing": { "type": "object", "required": [ "object", "model", "mode", "threshold", "split_percent", "updated_at" ], "properties": { "object": { "const": "routing" }, "model": { "type": "string" }, "mode": { "type": "string", "enum": [ "observe", "model", "split" ] }, "threshold": { "type": "number", "minimum": 0, "maximum": 1, "description": "Under this confidence a call falls back." }, "split_percent": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100 }, "updated_at": { "$ref": "#/components/schemas/Timestamp" } } }, "RoutingUpdate": { "type": "object", "additionalProperties": false, "properties": { "mode": { "type": "string", "enum": [ "observe", "model", "split" ] }, "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, "split_percent": { "type": "integer", "minimum": 0, "maximum": 100 } } }, "Model": { "type": "object", "required": [ "id", "object", "slug", "name", "decision", "status", "live_version", "answers", "outcomes", "metric", "lead", "routing", "experimenting", "budget", "owner", "calls_per_day", "created_at" ], "properties": { "id": { "type": "string", "example": "mdl_7f3a91c2" }, "object": { "const": "model" }, "slug": { "type": "string", "example": "grade-audit" }, "name": { "type": "string", "example": "Grade audit" }, "decision": { "type": "string", "description": "The decision the model makes, in one sentence." }, "status": { "$ref": "#/components/schemas/ModelStatus" }, "live_version": { "type": [ "string", "null" ], "example": "v7" }, "answers": { "type": "array", "items": { "$ref": "#/components/schemas/Answer" } }, "outcomes": { "type": "array", "description": "The outcomes you send back for its calls.", "items": { "type": "string" } }, "input_schema": { "type": [ "object", "null" ], "additionalProperties": true, "description": "JSON Schema for `input`, when the model takes structured cases." }, "metric": { "type": "string", "example": "Right verdict on the grade" }, "lead": { "oneOf": [ { "$ref": "#/components/schemas/Lead" }, { "type": "null" } ] }, "routing": { "$ref": "#/components/schemas/Routing" }, "experimenting": { "type": "boolean", "description": "Whether Pensieve's agents are running experiments on it. Pause and resume them with `models.update`." }, "budget": { "$ref": "#/components/schemas/ModelBudget" }, "owner": { "type": "string", "example": "Environments team" }, "calls_per_day": { "type": "integer" }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "ModelBudget": { "type": "object", "description": "Compute for experiments this month. Serving is not counted here.", "required": [ "monthly_usd", "spent_usd" ], "properties": { "monthly_usd": { "type": [ "number", "null" ], "description": "The cap; null for none." }, "spent_usd": { "type": "number" } } }, "ModelUpdate": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string", "minLength": 2, "maxLength": 80 }, "experimenting": { "type": "boolean", "description": "False pauses the agents: running experiments stop after their current runs, and no new ones start." }, "monthly_budget_usd": { "type": [ "number", "null" ], "minimum": 0 } } }, "ModelList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Model" } } } } ] }, "DecideRequest": { "type": "object", "additionalProperties": false, "required": [ "input" ], "properties": { "input": { "description": "The case, as the model's `input_schema` describes, or as text.", "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "key": { "type": "string", "maxLength": 255, "description": "Your own reference for the case, to send outcomes by." }, "version": { "type": "string", "description": "Pin a version instead of the live one." }, "metadata": { "type": "object", "additionalProperties": { "type": "string" }, "maxProperties": 20 } } }, "CallSource": { "type": "string", "enum": [ "pensieve", "capture" ], "description": "`pensieve`: an expert model answered. `capture`: your own code answered and the call was recorded." }, "Decision": { "type": "object", "required": [ "id", "object", "model", "version", "answer", "label", "confidence", "scores", "reason", "routed", "key", "latency_ms", "created_at" ], "properties": { "id": { "type": "string", "description": "The call id; send outcomes and corrections with it.", "example": "cl_8f21c0d4" }, "object": { "const": "decision" }, "model": { "type": "string", "example": "grade-audit" }, "version": { "type": "string", "example": "v7" }, "answer": { "type": "string", "description": "One of the model's answer values.", "example": "false_pass" }, "label": { "type": "string", "example": "False pass" }, "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, "scores": { "type": "object", "description": "The probability of each answer value.", "additionalProperties": { "type": "number" } }, "reason": { "type": [ "string", "null" ], "description": "A short reason, when the model gives one." }, "routed": { "type": "string", "enum": [ "model", "fallback" ], "description": "`fallback` when your code should answer instead: routing is `observe`, the confidence is under the threshold, or the split sent the call to your code." }, "key": { "type": [ "string", "null" ] }, "latency_ms": { "type": "integer" }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "Call": { "type": "object", "required": [ "id", "object", "model", "source", "input", "answer", "version", "confidence", "key", "outcome", "correction", "fallback", "created_at" ], "properties": { "id": { "type": "string" }, "object": { "const": "call" }, "model": { "type": "string" }, "source": { "$ref": "#/components/schemas/CallSource" }, "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "answer": { "type": "string" }, "version": { "type": [ "string", "null" ], "description": "The version that answered; null for captured calls." }, "confidence": { "type": [ "number", "null" ] }, "provider_model": { "type": [ "string", "null" ], "description": "For captured calls, the model your code called." }, "key": { "type": [ "string", "null" ] }, "outcome": { "oneOf": [ { "$ref": "#/components/schemas/Outcome" }, { "type": "null" } ] }, "correction": { "oneOf": [ { "$ref": "#/components/schemas/Correction" }, { "type": "null" } ] }, "fallback": { "description": "Your code's answer, when Pensieve routed this decision to your code.", "oneOf": [ { "$ref": "#/components/schemas/CallFallback" }, { "type": "null" } ] }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "CallFallback": { "type": "object", "required": [ "answer", "provider_model", "latency_ms", "recorded_at" ], "properties": { "answer": { "type": "string", "description": "The answer value your code gave." }, "provider_model": { "type": [ "string", "null" ], "description": "The model your code called, if any." }, "latency_ms": { "type": [ "integer", "null" ] }, "recorded_at": { "$ref": "#/components/schemas/Timestamp" } } }, "CallList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Call" } } } } ] }, "OutcomeCreate": { "type": "object", "additionalProperties": false, "required": [ "outcome" ], "properties": { "outcome": { "type": "string", "description": "One of the model's outcomes, such as `lab_rejected`." }, "value": { "type": "number", "description": "An amount that goes with it, such as money lost." }, "occurred_at": { "$ref": "#/components/schemas/Timestamp" }, "note": { "type": "string", "maxLength": 2000 } } }, "Outcome": { "type": "object", "required": [ "object", "call_id", "outcome", "value", "occurred_at", "note", "received_at" ], "properties": { "object": { "const": "outcome" }, "call_id": { "type": "string" }, "outcome": { "type": "string" }, "value": { "type": [ "number", "null" ] }, "occurred_at": { "$ref": "#/components/schemas/Timestamp" }, "note": { "type": [ "string", "null" ] }, "received_at": { "$ref": "#/components/schemas/Timestamp" } } }, "OutcomeBulk": { "type": "object", "additionalProperties": false, "required": [ "items" ], "properties": { "items": { "type": "array", "minItems": 1, "maxItems": 1000, "items": { "$ref": "#/components/schemas/OutcomeItem" } } } }, "OutcomeItem": { "type": "object", "additionalProperties": false, "required": [ "outcome" ], "properties": { "call_id": { "type": "string" }, "model": { "type": "string" }, "key": { "type": "string" }, "outcome": { "type": "string" }, "value": { "type": "number" }, "occurred_at": { "$ref": "#/components/schemas/Timestamp" }, "note": { "type": "string", "maxLength": 2000 } } }, "CorrectionCreate": { "type": "object", "additionalProperties": false, "required": [ "answer", "by" ], "properties": { "answer": { "type": "string", "description": "The right answer value." }, "by": { "type": "string", "description": "Who gave it, as an email." }, "note": { "type": "string", "maxLength": 2000 } } }, "Correction": { "type": "object", "required": [ "object", "call_id", "answer", "by", "note", "created_at" ], "properties": { "object": { "const": "correction" }, "call_id": { "type": "string" }, "answer": { "type": "string" }, "by": { "type": "string" }, "note": { "type": [ "string", "null" ] }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "CaptureCreate": { "type": "object", "additionalProperties": false, "required": [ "model", "calls" ], "properties": { "model": { "type": "string" }, "calls": { "type": "array", "minItems": 1, "maxItems": 1000, "items": { "$ref": "#/components/schemas/CaptureCall" } } } }, "CaptureCall": { "type": "object", "additionalProperties": false, "required": [ "input", "answer" ], "properties": { "id": { "type": "string", "description": "A `cl_` id you made, so you can send outcomes before this arrives." }, "decision_id": { "type": "string", "description": "The id of the decision this call follows, when Pensieve routed the case to your code. Your code's answer is recorded on that call instead of a new one. Don't send `id` with it." }, "key": { "type": "string" }, "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "answer": { "type": "string" }, "confidence": { "type": "number" }, "source": { "type": "string", "description": "What answered, such as `frontier`, `rules` or `human`.", "default": "frontier" }, "provider_model": { "type": "string" }, "latency_ms": { "type": "integer" }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "BatchCreate": { "type": "object", "additionalProperties": false, "required": [ "cases" ], "properties": { "cases": { "type": "array", "minItems": 1, "maxItems": 10000, "items": { "$ref": "#/components/schemas/BatchCase" } }, "version": { "type": "string" } } }, "BatchCase": { "type": "object", "additionalProperties": false, "required": [ "input" ], "properties": { "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "key": { "type": "string" } } }, "Batch": { "type": "object", "required": [ "id", "object", "model", "status", "total", "done", "decisions", "created_at", "completed_at" ], "properties": { "id": { "type": "string", "example": "bat_5d2e81f0" }, "object": { "const": "batch" }, "model": { "type": "string" }, "status": { "type": "string", "enum": [ "queued", "running", "done", "failed" ] }, "total": { "type": "integer" }, "done": { "type": "integer" }, "decisions": { "type": [ "array", "null" ], "description": "The decisions, in the order sent, once the batch is done.", "items": { "$ref": "#/components/schemas/Decision" } }, "created_at": { "$ref": "#/components/schemas/Timestamp" }, "completed_at": { "type": [ "string", "null" ], "format": "date-time" } } }, "ModelVersion": { "type": "object", "required": [ "id", "object", "model", "version", "status", "released_at", "note", "test_score", "frontier_score", "expert_hours", "size", "weights_available", "recipe" ], "properties": { "id": { "type": "string", "example": "ver_grade_audit_v7" }, "object": { "const": "version" }, "model": { "type": "string" }, "version": { "type": "string", "example": "v7" }, "status": { "type": "string", "enum": [ "candidate", "live", "standby", "retired" ] }, "released_at": { "type": [ "string", "null" ], "format": "date-time" }, "note": { "type": "string" }, "test_score": { "type": [ "number", "null" ] }, "frontier_score": { "type": [ "number", "null" ], "description": "The best frontier model's score when this version was tested." }, "expert_hours": { "type": "number", "description": "Expert time this version took." }, "size": { "type": "string", "example": "8B parameters" }, "weights_available": { "type": "boolean" }, "recipe": { "oneOf": [ { "$ref": "#/components/schemas/Recipe" }, { "type": "null" } ] } } }, "Recipe": { "type": "object", "description": "The post-training a version is made of. Each kept experiment changes one thing in it; the next experiment's A is this recipe.", "required": [ "base_model", "steps", "data", "experiments" ], "properties": { "base_model": { "type": "string", "description": "The open model it starts from.", "example": "Open 8B base" }, "steps": { "type": "array", "description": "The training steps, in order.", "items": { "$ref": "#/components/schemas/RecipeStep" } }, "data": { "type": "array", "description": "The data sources it learns from, and how each is used.", "items": { "$ref": "#/components/schemas/RecipeData" } }, "experiments": { "type": "array", "items": { "type": "string" }, "description": "The kept experiments that shaped this recipe." } } }, "RecipeStep": { "type": "object", "required": [ "method", "description" ], "properties": { "method": { "$ref": "#/components/schemas/MethodSlug" }, "description": { "type": "string", "example": "Weight overturned grades 4×" } } }, "RecipeData": { "type": "object", "required": [ "source", "use" ], "properties": { "source": { "type": "string", "description": "A source id.", "example": "src_reviewer_verdicts" }, "use": { "type": "string", "description": "How the recipe uses it.", "example": "Training examples; 5,000 held out as your test" } } }, "VersionList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ModelVersion" } } } } ] }, "Weights": { "type": "object", "required": [ "object", "model", "version", "format", "files", "expires_at" ], "properties": { "object": { "const": "weights" }, "model": { "type": "string" }, "version": { "type": "string" }, "format": { "type": "string", "enum": [ "safetensors" ] }, "files": { "type": "array", "items": { "$ref": "#/components/schemas/WeightsFile" } }, "expires_at": { "$ref": "#/components/schemas/Timestamp" } } }, "WeightsFile": { "type": "object", "required": [ "name", "size_bytes", "sha256", "url" ], "properties": { "name": { "type": "string" }, "size_bytes": { "type": "integer" }, "sha256": { "type": "string" }, "url": { "type": "string", "format": "uri" } } }, "Test": { "type": "object", "required": [ "object", "model", "metric", "cases", "scores", "updated_at" ], "properties": { "object": { "const": "test" }, "model": { "type": "string" }, "metric": { "type": "string" }, "cases": { "type": "integer", "description": "Held-out cases; no model ever trains on them." }, "scores": { "type": "array", "items": { "$ref": "#/components/schemas/TestScore" } }, "updated_at": { "$ref": "#/components/schemas/Timestamp" } } }, "TestScore": { "type": "object", "required": [ "subject", "kind", "score", "scored_at" ], "properties": { "subject": { "type": "string", "description": "A version such as `v7`, or a frontier or general model's name." }, "kind": { "type": "string", "enum": [ "version", "frontier", "general" ] }, "score": { "type": "number", "description": "On the test's metric, 0 to 1." }, "scored_at": { "$ref": "#/components/schemas/Timestamp" } } }, "TestCase": { "type": "object", "required": [ "id", "object", "model", "input", "answer", "by", "source", "added_at" ], "properties": { "id": { "type": "string", "example": "case_grade_0001" }, "object": { "const": "test_case" }, "model": { "type": "string" }, "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "answer": { "type": "string", "description": "The right answer value." }, "by": { "type": [ "string", "null" ], "description": "The expert who set it." }, "source": { "type": "string", "enum": [ "expert", "outcome", "review", "seed" ] }, "added_at": { "$ref": "#/components/schemas/Timestamp" } } }, "TestCaseList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/TestCase" } } } } ] }, "TestCasesAdd": { "type": "object", "additionalProperties": false, "required": [ "cases" ], "properties": { "cases": { "type": "array", "minItems": 1, "maxItems": 1000, "items": { "$ref": "#/components/schemas/NewTestCase" } } } }, "NewTestCase": { "type": "object", "additionalProperties": false, "required": [ "input", "answer" ], "properties": { "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" } ] }, "answer": { "type": "string" }, "by": { "type": "string" } } }, "ExperimentStatus": { "type": "string", "enum": [ "queued", "running", "kept", "dropped", "stopped" ] }, "MethodSlug": { "type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$", "maxLength": 64, "description": "A post-training method's slug, such as `fine-tuning`. See List methods for what your workspace has.", "example": "fine-tuning" }, "Method": { "type": "object", "description": "A post-training method, written as a plain markdown file. It starts with a `#` title and a paragraph\nsaying what the method does and when to try it; the rest is up to its author. Pensieve's own files have\nNeeds, Steps, Settings (what an experiment can change) and Checks. Agents read the file to design\nexperiments with the method and follow it as written.\n", "required": [ "id", "object", "name", "description", "markdown", "version", "built_in", "created_at", "updated_at" ], "properties": { "id": { "$ref": "#/components/schemas/MethodSlug" }, "object": { "const": "method" }, "name": { "type": "string", "description": "The file's title.", "example": "Fine-tuning" }, "description": { "type": "string", "description": "The file's first paragraph." }, "markdown": { "type": "string", "description": "The whole file." }, "version": { "type": "integer", "minimum": 1, "description": "Goes up by one with every change. Each experiment records the version it ran." }, "built_in": { "type": "boolean", "description": "Shipped by Pensieve. Built-in methods can't be changed or removed." }, "created_at": { "$ref": "#/components/schemas/Timestamp" }, "updated_at": { "$ref": "#/components/schemas/Timestamp" } } }, "MethodList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Method" } } } } ] }, "MethodCreate": { "type": "object", "additionalProperties": false, "required": [ "slug", "markdown" ], "properties": { "slug": { "$ref": "#/components/schemas/MethodSlug" }, "markdown": { "type": "string", "minLength": 1, "maxLength": 100000, "description": "The method, in plain markdown. It must start with a `#` title, then a paragraph." } } }, "MethodUpdate": { "type": "object", "additionalProperties": false, "required": [ "markdown" ], "properties": { "markdown": { "type": "string", "minLength": 1, "maxLength": 100000, "description": "The whole new file." } } }, "Experiment": { "type": "object", "description": "One controlled test of a change to the model's post-training. A is the current recipe; B is the current\nrecipe with one change. Both are trained on the same data and seeds and scored on your held-out test.\nA change is kept only if B beats A on every seed and on a last check against cases neither has seen.\n", "required": [ "id", "object", "model", "status", "idea", "hypothesis", "method", "method_version", "lever", "control", "variant", "prep", "seeds_done", "seeds_total", "gains", "lead_points", "throughput", "checks", "cost_usd", "max_cost_usd", "gpu_hours", "why", "result", "started_at", "ended_at" ], "properties": { "id": { "type": "string", "example": "exp_0420" }, "object": { "const": "experiment" }, "model": { "type": "string" }, "status": { "$ref": "#/components/schemas/ExperimentStatus" }, "idea": { "type": "string", "description": "The idea, in a line." }, "hypothesis": { "type": [ "string", "null" ], "description": "What the agents expect, and why it would beat the current recipe. Null until they design it." }, "method": { "oneOf": [ { "$ref": "#/components/schemas/MethodSlug" }, { "type": "null" } ], "description": "How B is trained. Null until the agents choose, if you didn't." }, "method_version": { "type": [ "integer", "null" ], "minimum": 1, "description": "The version of the method's file the experiment ran. Retrieve that version to read exactly what the agents followed." }, "lever": { "type": [ "string", "null" ], "description": "The part of the recipe the change pulls on, such as `Data mix`, `Reward`, `Synthetic data` or `Model size`.", "example": "Reward" }, "control": { "type": [ "string", "null" ], "description": "A: the current recipe." }, "variant": { "type": [ "string", "null" ], "description": "B: the current recipe with the change." }, "prep": { "type": [ "string", "null" ], "description": "The step that builds B's data or model before training, such as writing practice cases or building a reward." }, "seeds_done": { "type": "integer" }, "seeds_total": { "type": "integer" }, "gains": { "type": "array", "items": { "type": "number" }, "description": "B minus A on each finished seed, in points on your test." }, "lead_points": { "type": [ "number", "null" ], "description": "Kept: B's lead on the last check. Dropped: the mean of `gains`. Null while queued or running, and when stopped." }, "throughput": { "type": [ "number", "null" ], "description": "B's cases per second as a multiple of A's. Under 1 is slower to serve." }, "checks": { "type": "array", "items": { "type": "string" }, "description": "What was checked before the result counted, such as no test case leaking into training." }, "cost_usd": { "type": "number", "description": "Compute spent so far." }, "max_cost_usd": { "type": [ "number", "null" ], "description": "Its cost cap, if one was set. It stops before its compute would cost more." }, "gpu_hours": { "type": "number" }, "why": { "type": [ "string", "null" ], "description": "The evidence that led the agents to try it." }, "result": { "type": [ "string", "null" ], "description": "What happened: kept and part of a version, or dropped and why." }, "started_at": { "type": [ "string", "null" ], "format": "date-time" }, "ended_at": { "type": [ "string", "null" ], "format": "date-time" } } }, "ExperimentRun": { "type": "object", "description": "One step of an experiment. B's data is built once (`prep`), A and B are each trained on every seed (`train`), and a change that held on every seed gets a last check on cases neither has seen (`check`).", "required": [ "id", "object", "experiment", "kind", "arm", "seed", "label", "steps_done", "steps_total", "score", "lead_points", "gpu_hours", "status" ], "properties": { "id": { "type": "string", "example": "run_0420_b1" }, "object": { "const": "experiment_run" }, "experiment": { "type": "string", "example": "exp_0420" }, "kind": { "type": "string", "enum": [ "prep", "train", "check" ] }, "arm": { "type": [ "string", "null" ], "enum": [ "A", "B", null ], "description": "A, the current recipe, or B, the change; null for prep and the check." }, "seed": { "type": [ "integer", "null" ] }, "label": { "type": "string" }, "steps_done": { "type": [ "integer", "null" ], "description": "Training steps done, for a training run." }, "steps_total": { "type": [ "integer", "null" ] }, "score": { "type": [ "number", "null" ], "description": "The run's score on your test, 0 to 1, once it is done." }, "lead_points": { "type": [ "number", "null" ], "description": "For the check: B's lead over A on cases neither has seen." }, "gpu_hours": { "type": "number" }, "status": { "type": "string", "enum": [ "queued", "running", "done", "skipped" ] } } }, "ExperimentRunList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ExperimentRun" } } } } ] }, "ExperimentList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Experiment" } } } } ] }, "ExperimentCreate": { "type": "object", "additionalProperties": false, "required": [ "idea" ], "properties": { "idea": { "type": "string", "minLength": 3, "maxLength": 500, "description": "The idea, in a line. The agents write the hypothesis and design the runs." }, "method": { "$ref": "#/components/schemas/MethodSlug" }, "max_cost_usd": { "type": "number", "minimum": 1, "description": "Stop the experiment if its compute would cost more than this. It also counts against the model's monthly budget." } } }, "Gate": { "type": "object", "required": [ "kind", "label", "passed", "passed_at" ], "properties": { "kind": { "type": "string", "enum": [ "beats_live", "beats_frontier", "shadow", "signoff" ] }, "label": { "type": "string" }, "passed": { "type": "boolean" }, "passed_at": { "type": [ "string", "null" ], "format": "date-time" } } }, "Release": { "type": "object", "required": [ "id", "object", "model", "version", "status", "gates", "shadow", "split_percent", "signed_off_by", "created_at" ], "properties": { "id": { "type": "string", "example": "rel_grade_audit_v8" }, "object": { "const": "release" }, "model": { "type": "string" }, "version": { "type": "string" }, "status": { "type": "string", "enum": [ "pending", "shadow", "ready_for_signoff", "signed_off", "split", "live", "superseded", "rolled_back", "stopped" ], "description": "`superseded`: it went live and a later release replaced it." }, "gates": { "type": "array", "items": { "$ref": "#/components/schemas/Gate" } }, "shadow": { "oneOf": [ { "$ref": "#/components/schemas/ReleaseShadow" }, { "type": "null" } ] }, "split_percent": { "type": [ "integer", "null" ] }, "signed_off_by": { "type": [ "string", "null" ] }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "ReleaseShadow": { "type": "object", "required": [ "day", "days", "agreement" ], "properties": { "day": { "type": "integer" }, "days": { "type": "integer" }, "agreement": { "type": [ "number", "null" ], "description": "Share of shadow calls where it matched your experts or the outcome." } } }, "ReleaseList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Release" } } } } ] }, "ReleaseCreate": { "type": "object", "additionalProperties": false, "required": [ "version" ], "properties": { "version": { "type": "string" }, "shadow_days": { "type": "integer", "minimum": 1, "maximum": 30, "description": "The workspace's rule if left out." } } }, "SignOff": { "type": "object", "additionalProperties": false, "properties": { "by": { "type": "string", "description": "The expert signing off, as an email. The key's owner if left out." }, "note": { "type": "string", "maxLength": 2000 } } }, "SplitRequest": { "type": "object", "additionalProperties": false, "required": [ "percent" ], "properties": { "percent": { "type": "integer", "minimum": 1, "maximum": 99 } } }, "PromoteRequest": { "type": "object", "additionalProperties": false, "required": [ "expect" ], "properties": { "expect": { "type": [ "string", "null" ], "description": "The version you expect to be live now; null when none is." } } }, "RollbackRequest": { "type": "object", "additionalProperties": false, "properties": { "expect": { "type": "string", "description": "The version you expect to be live now." }, "reason": { "type": "string", "maxLength": 2000 } } }, "ReviewKind": { "type": "string", "enum": [ "unsure", "disagreement", "signoff" ], "description": "`unsure`: the model fell under its threshold. `disagreement`: the outcome went against the model. `signoff`: a release waits for an expert." }, "ReviewCase": { "type": "object", "required": [ "id", "object", "model", "kind", "state", "call_id", "release_id", "input", "model_answer", "confidence", "outcome", "answer", "answered_by", "created_at" ], "properties": { "id": { "type": "string", "example": "rev_grade_01" }, "object": { "const": "review_case" }, "model": { "type": "string" }, "kind": { "$ref": "#/components/schemas/ReviewKind" }, "state": { "type": "string", "enum": [ "open", "answered" ] }, "call_id": { "type": [ "string", "null" ] }, "release_id": { "type": [ "string", "null" ] }, "input": { "oneOf": [ { "type": "object", "additionalProperties": true }, { "type": "string" }, { "type": "null" } ] }, "model_answer": { "type": [ "string", "null" ] }, "confidence": { "type": [ "number", "null" ] }, "outcome": { "type": [ "string", "null" ] }, "answer": { "type": [ "string", "null" ] }, "answered_by": { "type": [ "string", "null" ] }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "ReviewCaseList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ReviewCase" } } } } ] }, "ReviewAnswer": { "type": "object", "additionalProperties": false, "required": [ "answer", "by" ], "properties": { "answer": { "type": "string" }, "by": { "type": "string" }, "note": { "type": "string", "maxLength": 2000 }, "add_to_test": { "type": "boolean", "default": false, "description": "Also hold the case out as a test case." } } }, "TaskCreate": { "type": "object", "additionalProperties": false, "required": [ "name", "decision", "answers" ], "properties": { "name": { "type": "string", "minLength": 2, "maxLength": 80 }, "decision": { "type": "string", "minLength": 10, "maxLength": 1000, "description": "The judgement call your software makes." }, "answers": { "type": "array", "minItems": 2, "maxItems": 20, "items": { "$ref": "#/components/schemas/TaskAnswer" } }, "outcomes": { "type": "array", "items": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "string" }, "description": "Ids of connected sources it learns from." }, "test": { "$ref": "#/components/schemas/TaskTest" }, "runs_in": { "type": "string", "enum": [ "your_vpc", "pensieve_cloud" ], "default": "your_vpc" }, "monthly_budget_usd": { "type": "number", "minimum": 0 }, "example": { "type": "string", "description": "One example case, as text." } } }, "TaskAnswer": { "type": "object", "additionalProperties": false, "required": [ "value" ], "properties": { "value": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, "label": { "type": "string" }, "description": { "type": "string" } } }, "TaskTest": { "type": "object", "additionalProperties": false, "properties": { "held_out_cases": { "type": "integer", "minimum": 100 }, "metric": { "type": "string" } } }, "Retest": { "type": "object", "required": [ "id", "object", "kind", "released_model", "released_at", "retested_at", "results" ], "properties": { "id": { "type": "string", "example": "rt_2026_10_22" }, "object": { "const": "retest" }, "kind": { "type": "string", "enum": [ "frontier", "general" ] }, "released_model": { "type": "string", "description": "The model that was released." }, "released_at": { "$ref": "#/components/schemas/Timestamp" }, "retested_at": { "$ref": "#/components/schemas/Timestamp" }, "results": { "type": "array", "items": { "$ref": "#/components/schemas/RetestResult" } } } }, "RetestResult": { "type": "object", "required": [ "model", "lead_before", "lead_after", "action" ], "properties": { "model": { "type": "string" }, "lead_before": { "type": "number" }, "lead_after": { "type": "number" }, "action": { "type": "string" } } }, "RetestList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Retest" } } } } ] }, "Source": { "type": "object", "required": [ "id", "object", "name", "kind", "via", "state", "volume", "held_out", "models", "last_read_at", "created_at" ], "properties": { "id": { "type": "string", "example": "src_agent_trials" }, "object": { "const": "source" }, "name": { "type": "string" }, "kind": { "type": "string", "enum": [ "stream", "database", "files", "webhook", "sdk" ] }, "via": { "type": "string", "example": "Kafka" }, "state": { "type": "string", "enum": [ "connecting", "connected", "paused", "error" ] }, "volume": { "type": [ "string", "null" ], "example": "240k a day" }, "held_out": { "type": [ "string", "null" ], "description": "What is kept out of every training set, such as the last 60 days, or the cases set aside as your test.", "example": "Last 60 days held out" }, "models": { "type": "array", "items": { "type": "string" } }, "last_read_at": { "type": [ "string", "null" ], "format": "date-time" }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "SourceList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Source" } } } } ] }, "SourceCreate": { "type": "object", "additionalProperties": false, "required": [ "name", "kind" ], "properties": { "name": { "type": "string" }, "kind": { "type": "string", "enum": [ "stream", "database", "files", "webhook", "sdk" ] }, "via": { "type": "string" }, "models": { "type": "array", "items": { "type": "string" } } } }, "SourceUpdate": { "type": "object", "additionalProperties": false, "required": [ "state" ], "properties": { "state": { "type": "string", "enum": [ "connected", "paused" ] } } }, "ApiKey": { "type": "object", "required": [ "id", "object", "name", "prefix", "created_at", "last_used_at", "revoked_at" ], "properties": { "id": { "type": "string", "example": "key_8c1f20" }, "object": { "const": "api_key" }, "name": { "type": "string" }, "prefix": { "type": "string", "example": "pk_live_7f3a" }, "secret": { "type": "string", "description": "The whole key, in the response that creates it only." }, "created_at": { "$ref": "#/components/schemas/Timestamp" }, "last_used_at": { "type": [ "string", "null" ], "format": "date-time" }, "revoked_at": { "type": [ "string", "null" ], "format": "date-time" } } }, "ApiKeyList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ApiKey" } } } } ] }, "ApiKeyCreate": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "name": { "type": "string", "minLength": 1, "maxLength": 80 } } }, "Member": { "type": "object", "required": [ "id", "object", "name", "email", "role", "state", "created_at" ], "properties": { "id": { "type": "string", "example": "mem_maya" }, "object": { "const": "member" }, "name": { "type": [ "string", "null" ] }, "email": { "type": "string" }, "role": { "type": "string", "enum": [ "admin", "member", "expert" ] }, "state": { "type": "string", "enum": [ "active", "invited" ] }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "MemberList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Member" } } } } ] }, "MemberInvite": { "type": "object", "additionalProperties": false, "required": [ "email", "role" ], "properties": { "email": { "type": "string", "format": "email" }, "role": { "type": "string", "enum": [ "admin", "member", "expert" ] } } }, "MemberUpdate": { "type": "object", "additionalProperties": false, "required": [ "role" ], "properties": { "role": { "type": "string", "enum": [ "admin", "member", "expert" ] } } }, "Usage": { "type": "object", "description": "Calls and compute for a month. Plans are priced per model per month, so nothing here is a bill.", "required": [ "object", "month", "models" ], "properties": { "object": { "const": "usage" }, "month": { "type": "string", "example": "2026-11" }, "models": { "type": "array", "items": { "$ref": "#/components/schemas/UsageModel" } } } }, "UsageModel": { "type": "object", "required": [ "model", "calls", "fallback_calls", "serving_cost_usd", "frontier_cost_usd", "experiments_usd", "experiments_budget_usd" ], "properties": { "model": { "type": "string" }, "calls": { "type": "integer" }, "fallback_calls": { "type": "integer", "description": "Calls your own code answered." }, "serving_cost_usd": { "type": "number", "description": "What serving the calls cost on the GPUs the model runs on." }, "frontier_cost_usd": { "type": "number", "description": "What the same calls would cost on the best frontier model." }, "experiments_usd": { "type": "number", "description": "Compute spent on experiments this month." }, "experiments_budget_usd": { "type": [ "number", "null" ], "description": "The monthly cap on experiment compute." } } }, "EventType": { "type": "string", "enum": [ "release.ready_for_signoff", "release.promoted", "release.rolled_back", "frontier.retested", "model.lead_narrowed", "experiment.ended", "review.case_added", "batch.completed" ] }, "Event": { "type": "object", "required": [ "id", "object", "type", "created_at", "data" ], "properties": { "id": { "type": "string", "example": "evt_1c9a3e" }, "object": { "const": "event" }, "type": { "$ref": "#/components/schemas/EventType" }, "created_at": { "$ref": "#/components/schemas/Timestamp" }, "data": { "type": "object", "additionalProperties": true, "description": "The object the event is about: a release, a retest, a model, an experiment, a review case or a batch." } } }, "EventList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Event" } } } } ] }, "Webhook": { "type": "object", "required": [ "id", "object", "url", "events", "created_at" ], "properties": { "id": { "type": "string", "example": "whe_3b7c11" }, "object": { "const": "webhook_endpoint" }, "url": { "type": "string", "format": "uri" }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/EventType" } }, "secret": { "type": "string", "description": "The signing secret, in the response that creates it only." }, "created_at": { "$ref": "#/components/schemas/Timestamp" } } }, "WebhookList": { "allOf": [ { "$ref": "#/components/schemas/ListMeta" }, { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Webhook" } } } } ] }, "WebhookCreate": { "type": "object", "additionalProperties": false, "required": [ "url", "events" ], "properties": { "url": { "type": "string", "format": "uri" }, "events": { "type": "array", "minItems": 1, "items": { "$ref": "#/components/schemas/EventType" } } } } } } }