{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://runpile.com/schemas/runpile-experiment-1.0.json",
  "title": "Runpile experiment manifest 1.0",
  "description": "A reproducible benchmark experiment containing one or more runs and sanitized evidence declarations.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "schema_version",
    "kind",
    "client_submission_id",
    "title",
    "visibility",
    "license",
    "runs",
    "provenance"
  ],
  "properties": {
    "schema_version": {
      "const": "1.0"
    },
    "kind": {
      "const": "experiment"
    },
    "client_submission_id": {
      "type": "string",
      "minLength": 8,
      "maxLength": 128
    },
    "title": {
      "type": "string",
      "minLength": 3,
      "maxLength": 180
    },
    "summary": {
      "type": "string",
      "maxLength": 1200
    },
    "objective": {
      "type": "string",
      "maxLength": 5000
    },
    "methodology": {
      "type": "string",
      "maxLength": 12000
    },
    "conclusions": {
      "type": "string",
      "maxLength": 12000
    },
    "limitations": {
      "type": "array",
      "maxItems": 100,
      "items": {
        "type": "string",
        "maxLength": 1000
      }
    },
    "visibility": {
      "$ref": "#/$defs/visibility"
    },
    "license": {
      "type": "string",
      "minLength": 1,
      "maxLength": 80
    },
    "tags": {
      "type": "array",
      "maxItems": 32,
      "uniqueItems": true,
      "items": {
        "type": "string",
        "minLength": 1,
        "maxLength": 48,
        "pattern": "^[a-z0-9][a-z0-9-]*$"
      }
    },
    "shared": {
      "$ref": "#/$defs/shared"
    },
    "runs": {
      "type": "array",
      "minItems": 1,
      "maxItems": 1000,
      "items": {
        "$ref": "#/$defs/run"
      }
    },
    "artifacts": {
      "type": "array",
      "maxItems": 500,
      "items": {
        "$ref": "#/$defs/artifact"
      }
    },
    "provenance": {
      "$ref": "#/$defs/provenance"
    },
    "publisher_notes": {
      "$ref": "#/$defs/publisherNotes"
    },
    "extensions": {
      "type": "object",
      "propertyNames": {
        "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
      },
      "additionalProperties": true
    }
  },
  "$defs": {
    "visibility": {
      "enum": [
        "public",
        "unlisted",
        "private"
      ]
    },
    "extensions": {
      "type": "object",
      "propertyNames": {
        "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
      },
      "additionalProperties": true
    },
    "artifact": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "client_artifact_id",
        "kind",
        "filename",
        "sha256"
      ],
      "properties": {
        "client_artifact_id": {
          "type": "string",
          "minLength": 1,
          "maxLength": 128,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
        },
        "kind": {
          "type": "string",
          "minLength": 1,
          "maxLength": 80
        },
        "filename": {
          "type": "string",
          "minLength": 1,
          "maxLength": 255,
          "pattern": "^[^/\\\\]+$"
        },
        "media_type": {
          "type": "string",
          "maxLength": 128
        },
        "size_bytes": {
          "type": "integer",
          "minimum": 0,
          "maximum": 26214400
        },
        "sha256": {
          "type": "string",
          "pattern": "^[a-fA-F0-9]{64}$"
        },
        "visibility": {
          "$ref": "#/$defs/visibility"
        },
        "raw_requests_included": {
          "type": "boolean"
        },
        "license": {
          "type": "string",
          "maxLength": 80
        },
        "run_client_id": {
          "type": "string",
          "maxLength": 128
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "metric": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "value",
        "measurement_status"
      ],
      "properties": {
        "value": {
          "type": "number"
        },
        "unit": {
          "type": "string",
          "maxLength": 48
        },
        "measurement_status": {
          "enum": [
            "measured",
            "derived",
            "inferred",
            "estimated"
          ]
        },
        "statistic": {
          "enum": [
            "total",
            "mean",
            "median",
            "min",
            "max",
            "p50",
            "p90",
            "p95",
            "p99",
            "stddev"
          ]
        },
        "scope": {
          "enum": [
            "run",
            "trial",
            "request_summary",
            "device"
          ]
        },
        "sample_count": {
          "type": "integer",
          "minimum": 0
        },
        "derivation": {
          "type": "string",
          "maxLength": 1000
        },
        "input_run_ids": {
          "type": "array",
          "items": {
            "type": "string",
            "maxLength": 128
          },
          "uniqueItems": true
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "measurement_status": {
                "enum": [
                  "derived",
                  "inferred",
                  "estimated"
                ]
              }
            },
            "required": [
              "measurement_status"
            ]
          },
          "then": {
            "required": [
              "derivation"
            ]
          }
        }
      ]
    },
    "tokenShape": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "kind"
      ],
      "properties": {
        "kind": {
          "enum": [
            "fixed",
            "distribution",
            "histogram",
            "artifact",
            "dataset_native"
          ]
        },
        "value": {
          "type": "integer",
          "minimum": 0
        },
        "mean": {
          "type": "number",
          "minimum": 0
        },
        "min": {
          "type": "integer",
          "minimum": 0
        },
        "max": {
          "type": "integer",
          "minimum": 0
        },
        "p50": {
          "type": "number",
          "minimum": 0
        },
        "p90": {
          "type": "number",
          "minimum": 0
        },
        "p95": {
          "type": "number",
          "minimum": 0
        },
        "p99": {
          "type": "number",
          "minimum": 0
        },
        "artifact_id": {
          "type": "string",
          "maxLength": 128
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "kind": {
                "const": "fixed"
              }
            },
            "required": [
              "kind"
            ]
          },
          "then": {
            "required": [
              "value"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "kind": {
                "const": "artifact"
              }
            },
            "required": [
              "kind"
            ]
          },
          "then": {
            "required": [
              "artifact_id"
            ]
          }
        }
      ]
    },
    "shared": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "environment": {
          "$ref": "#/$defs/environment"
        },
        "workload": {
          "$ref": "#/$defs/workload"
        },
        "measurement_method": {
          "$ref": "#/$defs/measurementMethod"
        }
      }
    },
    "provenance": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "source"
      ],
      "properties": {
        "source": {
          "enum": [
            "agent",
            "cli",
            "web",
            "api",
            "import",
            "other"
          ]
        },
        "tool": {
          "type": "string",
          "maxLength": 128
        },
        "tool_version": {
          "type": "string",
          "maxLength": 80
        },
        "generated_at": {
          "type": "string",
          "format": "date-time"
        },
        "original_experiment_id": {
          "type": "string",
          "maxLength": 128
        },
        "notes": {
          "type": "string",
          "maxLength": 2000
        }
      }
    },
    "publisherNotes": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "omitted_files": {
          "type": "array",
          "maxItems": 500,
          "items": {
            "type": "string",
            "maxLength": 255
          }
        },
        "redactions": {
          "type": "array",
          "maxItems": 500,
          "items": {
            "type": "string",
            "maxLength": 500
          }
        },
        "warnings": {
          "type": "array",
          "maxItems": 500,
          "items": {
            "type": "string",
            "maxLength": 1000
          }
        },
        "public_data_license_affirmed": {
          "type": "boolean"
        }
      }
    },
    "subject": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "type",
        "source",
        "identifier"
      ],
      "properties": {
        "type": {
          "enum": [
            "model",
            "other"
          ]
        },
        "source": {
          "enum": [
            "huggingface",
            "local",
            "api",
            "other"
          ]
        },
        "identifier": {
          "type": "string",
          "minLength": 1,
          "maxLength": 500
        },
        "revision": {
          "type": "string",
          "maxLength": 255
        },
        "revision_kind": {
          "enum": [
            "commit",
            "tag",
            "version",
            "digest",
            "unknown"
          ]
        },
        "architecture": {
          "type": "string",
          "maxLength": 255
        },
        "parameter_count_total": {
          "type": "number",
          "minimum": 0
        },
        "parameter_count_active": {
          "type": "number",
          "minimum": 0
        },
        "modalities": {
          "type": "array",
          "uniqueItems": true,
          "items": {
            "type": "string",
            "maxLength": 80
          }
        },
        "base_model": {
          "type": "string",
          "maxLength": 500
        },
        "quantization": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "method": {
              "enum": [
                "nvfp4",
                "fp4",
                "fp8",
                "int8",
                "int4",
                "awq",
                "gptq",
                "gguf",
                "none",
                "mixed",
                "other"
              ]
            },
            "format": {
              "enum": [
                "compressed-tensors",
                "modelopt",
                "gguf",
                "custom",
                "other"
              ]
            },
            "weight_dtype": {
              "type": "string",
              "maxLength": 64
            },
            "activation_dtype": {
              "type": "string",
              "maxLength": 64
            },
            "kv_cache_dtype": {
              "type": "string",
              "maxLength": 64
            },
            "scope": {
              "enum": [
                "all_linears",
                "experts_only",
                "mixed",
                "unknown"
              ]
            },
            "group_size": {
              "type": "integer",
              "minimum": 1
            },
            "block_size": {
              "type": "integer",
              "minimum": 1
            },
            "provider": {
              "type": "string",
              "maxLength": 128
            },
            "recipe": {
              "type": "string",
              "maxLength": 500
            },
            "extensions": {
              "type": "object",
              "propertyNames": {
                "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
              },
              "additionalProperties": true
            }
          }
        },
        "adapters": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": true
          }
        },
        "modifications": {
          "type": "array",
          "items": {
            "type": "string",
            "maxLength": 1000
          }
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "accelerator": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "vendor",
        "model",
        "count"
      ],
      "properties": {
        "vendor": {
          "type": "string",
          "minLength": 1,
          "maxLength": 80
        },
        "model": {
          "type": "string",
          "minLength": 1,
          "maxLength": 255
        },
        "architecture": {
          "type": "string",
          "maxLength": 128
        },
        "count": {
          "type": "integer",
          "minimum": 1,
          "maximum": 4096
        },
        "device_indices": {
          "type": "array",
          "uniqueItems": true,
          "items": {
            "type": "integer",
            "minimum": 0
          }
        },
        "memory_bytes_per_device": {
          "type": "integer",
          "minimum": 0
        },
        "driver": {
          "type": "string",
          "maxLength": 128
        },
        "firmware": {
          "type": "string",
          "maxLength": 128
        },
        "power_limit_watts": {
          "type": "number",
          "minimum": 0
        },
        "clock_settings": {
          "type": "string",
          "maxLength": 255
        },
        "partition": {
          "type": "string",
          "maxLength": 128
        },
        "interconnect": {
          "type": "string",
          "maxLength": 255
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "environment": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "os": {
          "type": "string",
          "maxLength": 128
        },
        "kernel": {
          "type": "string",
          "maxLength": 255
        },
        "host_architecture": {
          "type": "string",
          "maxLength": 64
        },
        "cpu": {
          "type": "object",
          "additionalProperties": true
        },
        "system_memory_bytes": {
          "type": "integer",
          "minimum": 0
        },
        "accelerators": {
          "type": "array",
          "minItems": 1,
          "items": {
            "$ref": "#/$defs/accelerator"
          }
        },
        "storage": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": true
          }
        },
        "cloud": {
          "type": "object",
          "additionalProperties": true
        },
        "container": {
          "type": "object",
          "additionalProperties": true
        },
        "virtual_environment": {
          "type": "object",
          "additionalProperties": true
        },
        "driver_versions": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          }
        },
        "topology": {
          "type": "string",
          "maxLength": 4000
        },
        "safe_environment": {
          "type": "object",
          "additionalProperties": {
            "type": "string",
            "maxLength": 1000
          }
        },
        "hardware_fingerprint": {
          "type": "string",
          "maxLength": 255
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "runtime": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "engine"
      ],
      "properties": {
        "engine": {
          "type": "string",
          "minLength": 1,
          "maxLength": 128
        },
        "version": {
          "type": "string",
          "maxLength": 128
        },
        "source_revision": {
          "type": "string",
          "maxLength": 255
        },
        "build_identifier": {
          "type": "string",
          "maxLength": 255
        },
        "container": {
          "type": "object",
          "additionalProperties": true
        },
        "command_sanitized": {
          "type": "string",
          "maxLength": 12000
        },
        "parallelism": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "tensor": {
              "type": "integer",
              "minimum": 1
            },
            "pipeline": {
              "type": "integer",
              "minimum": 1
            },
            "data": {
              "type": "integer",
              "minimum": 1
            },
            "expert": {
              "type": "integer",
              "minimum": 1
            },
            "replicas": {
              "type": "integer",
              "minimum": 1
            }
          }
        },
        "parameters": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9][a-z0-9_-]*$"
          },
          "additionalProperties": {
            "type": "object",
            "additionalProperties": true
          }
        },
        "safe_environment": {
          "type": "object",
          "additionalProperties": {
            "type": "string",
            "maxLength": 1000
          }
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "workload": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "task",
        "mode"
      ],
      "properties": {
        "task": {
          "enum": [
            "text_generation",
            "embedding",
            "reranking",
            "classification",
            "transcription",
            "image_generation",
            "other"
          ]
        },
        "mode": {
          "enum": [
            "online",
            "offline",
            "batch"
          ]
        },
        "dataset": {
          "type": "object",
          "additionalProperties": true
        },
        "request_count": {
          "type": "integer",
          "minimum": 0
        },
        "warmup_request_count": {
          "type": "integer",
          "minimum": 0
        },
        "concurrency": {
          "type": "integer",
          "minimum": 1
        },
        "request_rate_per_second": {
          "type": [
            "number",
            "null"
          ],
          "exclusiveMinimum": 0
        },
        "arrival_process": {
          "enum": [
            "closed_loop",
            "poisson",
            "burst",
            "trace",
            "sequential",
            "custom"
          ]
        },
        "streaming": {
          "type": "boolean"
        },
        "input_tokens": {
          "$ref": "#/$defs/tokenShape"
        },
        "output_tokens": {
          "$ref": "#/$defs/tokenShape"
        },
        "sampling": {
          "type": "object",
          "additionalProperties": true
        },
        "stopping": {
          "type": "object",
          "additionalProperties": true
        },
        "prefix_behavior": {
          "type": "object",
          "additionalProperties": true
        },
        "tokenizer": {
          "type": "object",
          "additionalProperties": true
        },
        "endpoint_type": {
          "type": "string",
          "maxLength": 128
        },
        "client": {
          "type": "object",
          "additionalProperties": true
        },
        "random_seed": {
          "type": "integer"
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "measurementMethod": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "harness": {
          "type": "string",
          "maxLength": 255
        },
        "version": {
          "type": "string",
          "maxLength": 128
        },
        "revision": {
          "type": "string",
          "maxLength": 255
        },
        "command_sanitized": {
          "type": "string",
          "maxLength": 12000
        },
        "client_environment": {
          "type": "object",
          "additionalProperties": true
        },
        "clock_source": {
          "type": "string",
          "maxLength": 128
        },
        "tokenization_included": {
          "type": "boolean"
        },
        "model_loading_included": {
          "type": "boolean"
        },
        "client_overhead_included": {
          "type": "boolean"
        },
        "warmup_semantics": {
          "type": "string",
          "maxLength": 2000
        },
        "retry_failure_treatment": {
          "type": "string",
          "maxLength": 2000
        },
        "duration_semantics": {
          "type": "string",
          "maxLength": 2000
        },
        "metric_definition_version": {
          "type": "string",
          "maxLength": 128
        },
        "notes": {
          "type": "string",
          "maxLength": 5000
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      }
    },
    "failure": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "stage",
        "category",
        "message_sanitized"
      ],
      "properties": {
        "stage": {
          "type": "string",
          "maxLength": 128
        },
        "category": {
          "type": "string",
          "maxLength": 128
        },
        "message_sanitized": {
          "type": "string",
          "maxLength": 4000
        },
        "exit_code": {
          "type": "integer"
        },
        "reproducible": {
          "type": "boolean"
        },
        "replacement_client_run_id": {
          "type": "string",
          "maxLength": 128
        }
      }
    },
    "trial": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "client_trial_id",
        "status"
      ],
      "properties": {
        "client_trial_id": {
          "type": "string",
          "maxLength": 128
        },
        "status": {
          "enum": [
            "completed",
            "failed",
            "interrupted",
            "partial",
            "invalid"
          ]
        },
        "started_at": {
          "type": "string",
          "format": "date-time"
        },
        "ended_at": {
          "type": "string",
          "format": "date-time"
        },
        "duration_seconds": {
          "type": "number",
          "minimum": 0
        },
        "results": {
          "type": "object",
          "additionalProperties": {
            "oneOf": [
              {
                "type": "number"
              },
              {
                "$ref": "#/$defs/metric"
              }
            ]
          }
        },
        "artifacts": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/artifact"
          }
        },
        "notes": {
          "type": "string",
          "maxLength": 4000
        }
      }
    },
    "run": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "client_run_id",
        "name",
        "status",
        "subject",
        "runtime"
      ],
      "properties": {
        "client_run_id": {
          "type": "string",
          "minLength": 1,
          "maxLength": 128,
          "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
        },
        "name": {
          "type": "string",
          "minLength": 1,
          "maxLength": 255
        },
        "description": {
          "type": "string",
          "maxLength": 4000
        },
        "status": {
          "enum": [
            "completed",
            "failed",
            "interrupted",
            "partial",
            "invalid"
          ]
        },
        "started_at": {
          "type": "string",
          "format": "date-time"
        },
        "ended_at": {
          "type": "string",
          "format": "date-time"
        },
        "duration_seconds": {
          "type": "number",
          "minimum": 0
        },
        "relation": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "type"
          ],
          "properties": {
            "type": {
              "enum": [
                "baseline",
                "sweep_point",
                "repetition",
                "replacement",
                "aggregate",
                "other"
              ]
            },
            "group": {
              "type": "string",
              "maxLength": 128
            },
            "replaces_client_run_id": {
              "type": "string",
              "maxLength": 128
            }
          }
        },
        "subject": {
          "$ref": "#/$defs/subject"
        },
        "environment": {
          "$ref": "#/$defs/environment"
        },
        "runtime": {
          "$ref": "#/$defs/runtime"
        },
        "workload": {
          "$ref": "#/$defs/workload"
        },
        "measurement_method": {
          "$ref": "#/$defs/measurementMethod"
        },
        "results": {
          "type": "object",
          "additionalProperties": {
            "oneOf": [
              {
                "type": "number"
              },
              {
                "$ref": "#/$defs/metric"
              }
            ]
          }
        },
        "trials": {
          "type": "array",
          "maxItems": 10000,
          "items": {
            "$ref": "#/$defs/trial"
          }
        },
        "failure": {
          "$ref": "#/$defs/failure"
        },
        "artifacts": {
          "type": "array",
          "maxItems": 500,
          "items": {
            "$ref": "#/$defs/artifact"
          }
        },
        "warnings": {
          "type": "array",
          "maxItems": 100,
          "items": {
            "type": "string",
            "maxLength": 1000
          }
        },
        "extensions": {
          "type": "object",
          "propertyNames": {
            "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$"
          },
          "additionalProperties": true
        }
      },
      "allOf": [
        {
          "if": {
            "properties": {
              "status": {
                "const": "failed"
              }
            },
            "required": [
              "status"
            ]
          },
          "then": {
            "required": [
              "failure"
            ]
          }
        },
        {
          "if": {
            "properties": {
              "status": {
                "const": "completed"
              }
            },
            "required": [
              "status"
            ]
          },
          "then": {
            "required": [
              "results"
            ]
          }
        }
      ]
    }
  }
}