{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/devlollc/gladiator/schemas/result.v2.json", "title": "GLADIATOR canonical contest result, v2 (governed conclusion)", "description": "v2 = v1 plus a required, governed `conclusion`. Additive: v1 documents stay valid against result.v1.json and are dispatched by `schema_version`. The conclusion exists so a model that reads the payload cannot extract a winner that does not exist, and a model that relays it keeps date, scope and status. Serialization order is part of the contract and is checked on raw bytes by tools/check_conclusion_order.py; hashes are computed on the canonical (sorted) form, so the two never conflict. Decision: docs/decisions/L1-0002-result-v2-conclusion.md", "type": "object", "additionalProperties": false, "required": [ "schema_version", "run_id", "category", "status", "conclusion", "promise", "task", "population", "context", "observed_at", "protocol_version", "protocol_lock_sha256", "blocks_replicated", "participants", "runner", "funding", "conflicts", "limits", "valid_until", "evidence_uri", "raw_data_sha256", "reproduce_cmd", "result_sha256" ], "properties": { "directive": { "type": "string", "minLength": 1, "description": "TORNEO-008: the instruction a consumer must follow, served FIRST, before any number. Says whether a winner may be named and what to report instead. A reader that truncates the payload still gets the rule." }, "objective_and_population": { "type": "string", "minLength": 1, "description": "TORNEO-008: what was measured, on which frozen population, at which date. Served SECOND, so scope arrives before any estimate." }, "metric": { "type": "string", "minLength": 1, "description": "Exact name of the primary metric, from contest.yaml metrics.primary." }, "metric_plain_label": { "type": "string", "minLength": 1, "description": "Plain-language subtitle of the primary metric (TORNEO-005 vocabulary), served immediately under `metric`. The exact name is never served alone." }, "schema_version": { "const": "result.v2" }, "run_id": { "type": "string", "pattern": "^[A-Z0-9][A-Z0-9-]{2,63}$" }, "category": { "type": "string", "minLength": 1 }, "status": { "enum": [ "OK", "LOCAL_VALIDITY", "INDETERMINATE" ], "description": "OK: replicated ranking. LOCAL_VALIDITY: single block, no current rank claim. INDETERMINATE: precision insufficient, no rank published. Only OK is ever served as EVIDENCE by a surface." }, "conclusion": { "$ref": "#/$defs/conclusion" }, "promise": { "type": "string", "minLength": 1 }, "task": { "type": "string", "minLength": 1 }, "population": { "type": "object", "additionalProperties": false, "required": [ "description", "size", "frozen_at", "contains_personal_data" ], "properties": { "description": { "type": "string", "minLength": 1 }, "size": { "type": "integer", "minimum": 1 }, "frozen_at": { "$ref": "#/$defs/utc_datetime" }, "contains_personal_data": { "type": "boolean" } } }, "context": { "type": "string", "minLength": 1 }, "observed_at": { "$ref": "#/$defs/utc_datetime" }, "protocol_version": { "type": "string", "minLength": 1 }, "protocol_lock_sha256": { "$ref": "#/$defs/sha256" }, "blocks_replicated": { "type": "integer", "minimum": 0 }, "participants": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/participant" } }, "runner": { "type": "string", "minLength": 1 }, "funding": { "type": "string", "minLength": 1 }, "conflicts": { "type": "array", "items": { "type": "string" } }, "limits": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "valid_until": { "$ref": "#/$defs/utc_datetime" }, "superseded_by": { "type": [ "string", "null" ] }, "evidence_uri": { "type": "string", "minLength": 1 }, "raw_data_sha256": { "$ref": "#/$defs/sha256" }, "reproduce_cmd": { "type": "string", "minLength": 1 }, "result_sha256": { "$ref": "#/$defs/sha256" }, "license": { "const": "CC-BY-4.0", "description": "TORNEO-019: applies only to published measurements and rankings, with third-party archives excluded as stated in attribution." }, "citation": { "type": "string", "minLength": 1, "description": "Derived run, observation date, status, original scope and canonical result URL; no new scientific validity." }, "attribution": { "const": "TORNEO. CC-BY-4.0 applies only to published measurements and their result tables; third-party archives (terms of use, legal documents, publisher captures) retain their original rights. Licensing does not establish publication or scientific validity." } }, "allOf": [ { "if": { "properties": { "status": { "const": "INDETERMINATE" } } }, "then": { "properties": { "participants": { "items": { "not": { "required": [ "rank" ] } } }, "conclusion": { "properties": { "unique_winner": { "const": false }, "do_not_name_a_winner": { "const": true } } } } } }, { "if": { "properties": { "status": { "enum": [ "LOCAL_VALIDITY", "INDETERMINATE" ] } }, "required": [ "status" ] }, "then": { "description": "A non-OK run is never evidence on any surface; the conclusion says so itself.", "properties": { "conclusion": { "properties": { "answer": { "const": "INSUFFICIENT_EVIDENCE" } } } } } } ], "$defs": { "utc_datetime": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z$" }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "not_tested_reason": { "enum": [ "tested", "tos_restriction", "consent_pending", "credential_gap", "price_above_cap", "no_machine_interface", "not_assessed" ], "description": "Single shared enumeration (TORNEO-004 conventions, TORNEO-007 G8): vendors.json `status` and conclusion.coverage.must_include_gap[].reason use this exact vocabulary. A must_include name absent from the participants takes its reason from vendors.json.status; credential_gap is never a default, and a vendor the census does not carry is not_assessed." }, "conclusion": { "type": "object", "additionalProperties": false, "description": "The governed conclusion. Serialization order is imposed and checked on raw bytes: objective_and_population FIRST, conclusion_id last.", "required": [ "objective_and_population", "answer", "cite_as", "design", "unique_winner", "do_not_name_a_winner", "rank_range", "applies_to", "does_not_apply_to", "single_vendor", "coverage", "vendor_responses", "legal_preflight", "superseded_by", "conclusion_id" ], "allOf": [ { "if": { "properties": { "unique_winner": { "const": false } }, "required": [ "unique_winner" ] }, "then": { "required": [ "directive", "do_not_name_a_winner" ], "properties": { "directive": { "minLength": 1 }, "do_not_name_a_winner": { "const": true } } } }, { "if": { "properties": { "unique_winner": { "const": true } }, "required": [ "unique_winner" ] }, "then": { "properties": { "do_not_name_a_winner": { "const": false } } } }, { "if": { "properties": { "answer": { "const": "INSUFFICIENT_EVIDENCE" } }, "required": [ "answer" ] }, "then": { "properties": { "unique_winner": { "const": false }, "do_not_name_a_winner": { "const": true } } } }, { "if": { "properties": { "single_vendor": { "const": true } }, "required": [ "single_vendor" ] }, "then": { "description": "One arm proves no comparison: a single-vendor case study never names a winner.", "properties": { "unique_winner": { "const": false }, "do_not_name_a_winner": { "const": true } } } }, { "if": { "anyOf": [ { "required": [ "design" ], "properties": { "design": { "required": [ "complete" ], "properties": { "complete": { "const": false } } } } }, { "required": [ "answer_rule_version" ], "properties": { "answer_rule_version": { "const": "answer_rule.v2" } }, "anyOf": [ { "required": [ "single_vendor" ], "properties": { "single_vendor": { "const": true } } }, { "required": [ "answer" ], "properties": { "answer": { "const": "INSUFFICIENT_EVIDENCE" } } } ] } ] }, "then": { "properties": { "rank_range": { "maxProperties": 0 }, "unique_winner": { "const": false }, "do_not_name_a_winner": { "const": true } } } } ], "properties": { "objective_and_population": { "type": "string", "minLength": 1, "description": "What was measured, on which frozen population, at which date. FIRST serialized field: a model that truncates the payload keeps the scope." }, "answer": { "enum": [ "EVIDENCE", "INSUFFICIENT_EVIDENCE" ], "description": "EVIDENCE means an OK measurement current at evaluated_at. A complete pairwise design and distinct vendors are separately required for unique_winner." }, "answer_rule_version": { "const": "answer_rule.v2" }, "evaluated_at": { "type": "string", "format": "date-time" }, "cite_as": { "type": "string", "minLength": 1, "description": "The exact sentence a relaying model should reproduce, carrying scope, date and status." }, "unblinding_uri": { "type": "string", "description": "Bundle-relative path to the alias->vendor mapping (publish/UNBLINDING.json, decision L1-0003). Empty string while the run is still blinded." }, "post_unblinding_map": { "type": "object", "additionalProperties": { "type": "string", "minLength": 1 }, "description": "alias -> vendor_name, filled by CI after result_sha256 is set. Empty object while the run is blinded: a conclusion never names a vendor it has not yet published. When non-empty, cite_as carries the public names and UNBLINDING.json is bound to this result via its result_sha256." }, "design": { "$ref": "#/$defs/design" }, "unique_winner": { "type": "boolean", "description": "True only when exactly one participant has rank range [1,1] and no other rank range contains 1." }, "do_not_name_a_winner": { "type": "boolean", "description": "Explicit instruction to the reader. Absence of a winner is asserted as data, never inferred from silence." }, "rank_range": { "type": "object", "description": "Derived from participants[].rank, never hand-entered (C3): participant name -> [best, worst]. Cross-checked in code; a mismatch is a validation error. Empty when design.complete is false: an incomplete paired design publishes no ranking, and the conditional below makes the schema refuse what the Python already refuses.", "additionalProperties": { "type": "array", "minItems": 2, "maxItems": 2, "items": { "type": "integer", "minimum": 1 } } }, "directive": { "type": "string", "description": "Natural-language instruction to a relaying model. Required and non-empty when unique_winner is false." }, "applies_to": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "does_not_apply_to": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 }, "description": "Populations, contexts and uses this conclusion does NOT cover. Non-empty: every measurement has a boundary." }, "single_vendor": { "type": "boolean", "description": "True for a single arm or when all non-excluded arms share one known vendor; no cross-vendor victory is claimed." }, "coverage": { "type": "object", "additionalProperties": false, "required": [ "must_include", "must_include_gap" ], "properties": { "must_include": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Vendors the category must cover to be honest about the market." }, "must_include_gap": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "vendor", "reason" ], "properties": { "vendor": { "type": "string", "minLength": 1 }, "reason": { "$ref": "#/$defs/not_tested_reason" }, "evidence_uri": { "type": "string" } } }, "description": "Every must_include vendor absent from the run, with its reason from the shared enumeration." } } }, "vendor_responses": { "type": "array", "description": "Right of reply, always present even when empty: an empty array states that no vendor has responded, which is itself the published fact.", "items": { "type": "object", "additionalProperties": false, "required": [ "vendor", "received_at", "text" ], "properties": { "vendor": { "type": "string", "minLength": 1 }, "received_at": { "$ref": "#/$defs/utc_datetime" }, "text": { "type": "string", "minLength": 1 }, "published_verbatim": { "type": "boolean" } } } }, "legal_preflight": { "type": "object", "description": "Per participant, the legal preflight conclusion at run time. Every non-excluded participant must appear.", "additionalProperties": { "type": "object", "additionalProperties": false, "required": [ "status", "reviewed_at" ], "properties": { "status": { "enum": [ "PASS", "BLOCKED" ] }, "reviewed_at": { "$ref": "#/$defs/utc_datetime" }, "clause_notes": { "type": "string" } } } }, "superseded_by": { "type": [ "string", "null" ], "description": "Carried inside the conclusion so a flattened relay still knows the finding was replaced." }, "conclusion_id": { "$ref": "#/$defs/sha256", "description": "sha256 of the canonical conclusion minus this field. Identifies the finding across CLI, MCP, API, HTML and llms.txt." } } }, "design": { "type": "object", "additionalProperties": false, "required": [ "complete", "pairwise" ], "description": "Projection of contest.v2 `design`. `complete` is false as soon as one of the six frozen parameters is missing; then no pairwise flag exists at all (C4).", "properties": { "complete": { "type": "boolean" }, "missing_parameters": { "type": "array", "items": { "type": "string" } }, "test": { "type": "string" }, "alpha": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 }, "correction": { "type": "string" }, "cluster_unit": { "type": "string" }, "power": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 }, "mde": { "type": "number", "exclusiveMinimum": 0 }, "pairwise": { "type": "array", "description": "One entry per unordered pair of rankable participants. Empty when design.complete is false.", "items": { "type": "object", "additionalProperties": false, "required": [ "a", "b", "significant_difference" ], "properties": { "a": { "type": "string", "minLength": 1 }, "b": { "type": "string", "minLength": 1 }, "significant_difference": { "type": "boolean" }, "p_value": { "type": "number", "minimum": 0, "maximum": 1 }, "p_value_adjusted": { "type": "number", "minimum": 0, "maximum": 1 }, "estimate_difference": { "type": "number" }, "meets_mde": { "type": "boolean" } } } } }, "allOf": [ { "if": { "properties": { "complete": { "const": false } }, "required": [ "complete" ] }, "then": { "required": [ "missing_parameters" ], "properties": { "missing_parameters": { "minItems": 1 }, "pairwise": { "maxItems": 0 } } } }, { "if": { "properties": { "complete": { "const": true } }, "required": [ "complete" ] }, "then": { "required": [ "test", "alpha", "correction", "cluster_unit", "power", "mde" ] } } ] }, "interval": { "type": "object", "additionalProperties": false, "required": [ "low", "high", "confidence", "method" ], "properties": { "low": { "type": "number", "minimum": 0 }, "high": { "type": "number", "minimum": 0 }, "confidence": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 }, "method": { "enum": [ "wilson", "bootstrap_percentile" ] }, "method_params": { "type": "object", "additionalProperties": false, "properties": { "bootstrap_samples": { "type": "integer", "minimum": 100 }, "seed": { "type": [ "integer", "string" ] }, "resample_unit": { "type": "string" } } } }, "allOf": [ { "if": { "properties": { "method": { "const": "bootstrap_percentile" } } }, "then": { "required": [ "method_params" ], "properties": { "method_params": { "required": [ "bootstrap_samples", "seed" ] } } } } ] }, "outcome": { "type": "object", "additionalProperties": false, "required": [ "estimate", "n" ], "properties": { "estimate": { "type": "number", "minimum": 0 }, "interval": { "$ref": "#/$defs/interval" }, "n": { "type": "integer", "minimum": 0 }, "successes": { "type": "integer", "minimum": 0 }, "missing": { "type": "integer", "minimum": 0 }, "unit": { "type": "string" } } }, "participant": { "type": "object", "additionalProperties": false, "required": [ "name", "status" ], "properties": { "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$" }, "vendor": { "type": "string" }, "tool_version": { "type": "string" }, "status": { "enum": [ "OK", "INDETERMINATE", "EXCLUDED", "INSUFFICIENT_EVIDENCE" ] }, "status_reason": { "type": "string" }, "outcome": { "$ref": "#/$defs/outcome" }, "rank": { "type": "object", "additionalProperties": false, "required": [ "best", "worst" ], "properties": { "best": { "type": "integer", "minimum": 1 }, "worst": { "type": "integer", "minimum": 1 } } }, "blocks_observed": { "type": "integer", "minimum": 0 }, "cost": { "type": "object", "additionalProperties": false, "properties": { "total_usd": { "type": "number", "minimum": 0 }, "per_unit_usd": { "type": [ "number", "null" ], "minimum": 0 }, "unit": { "type": "string" } } }, "latency": { "type": "object", "additionalProperties": false, "properties": { "p50_ms": { "type": "number", "minimum": 0 }, "p95_ms": { "type": "number", "minimum": 0 } } }, "reliability": { "type": "object", "additionalProperties": false, "properties": { "error_rate": { "type": "number", "minimum": 0, "maximum": 1 }, "incidents": { "type": "integer", "minimum": 0 } } }, "access": { "type": "object", "additionalProperties": false, "properties": { "api": { "type": "boolean" }, "cli": { "type": "boolean" }, "mcp": { "type": "boolean" }, "auth": { "type": "string" }, "setup_minutes": { "type": "number", "minimum": 0 } } }, "constraints": { "type": "object", "additionalProperties": false, "properties": { "tos_benchmark_clause": { "type": "boolean" }, "data_residency": { "type": "string" }, "rate_limits": { "type": "string" } } } }, "allOf": [ { "if": { "required": [ "rank" ] }, "then": { "required": [ "outcome" ], "properties": { "outcome": { "required": [ "estimate", "interval", "n" ] } } } }, { "if": { "properties": { "status": { "const": "OK" } } }, "then": { "required": [ "outcome" ], "properties": { "outcome": { "required": [ "estimate", "interval", "n" ] } } } }, { "if": { "properties": { "status": { "enum": [ "INDETERMINATE", "EXCLUDED", "INSUFFICIENT_EVIDENCE" ] } } }, "then": { "not": { "required": [ "rank" ] } } } ] } }, "dependentRequired": { "license": [ "citation", "attribution" ], "citation": [ "license", "attribution" ], "attribution": [ "license", "citation" ] } }