unsloth/gemma-4-26B-A4B-it-NVFP4

{
  "harness": "vllm bench serve",
  "version": "0.27.1",
  "clock_source": "CLOCK_MONOTONIC",
  "warmup_semantics": "Eight requests completed before the timed batch.",
  "duration_semantics": "First timed request dispatch through final response completion.",
  "tokenization_included": true,
  "model_loading_included": false,
  "retry_failure_treatment": "Failed requests remain in the failure count and are not retried.",
  "client_overhead_included": true,
  "metric_definition_version": "runpile-generation-1"
}

{
  "name": "Scheduler token budget 11264",
  "status": "completed",
  "trials": [
    {
      "status": "completed",
      "results": {
        "requests_per_second": {
          "unit": "req/s",
          "value": 8.52,
          "measurement_status": "measured"
        }
      },
      "artifacts": [
        {
          "kind": "benchmark_result_json",
          "sha256": "9ca84451584837616509201c58a0b1b9ee9ed44fec11a84c4ec6629f3cb568c4",
          "license": "CC-BY-4.0",
          "filename": "scheduler-11264.json",
          "media_type": "application/json",
          "size_bytes": 18432,
          "visibility": "public",
          "run_client_id": "scheduler-11264",
          "client_artifact_id": "scheduler-11264-raw",
          "raw_requests_included": false
        }
      ],
      "client_trial_id": "scheduler-11264-trial-1",
      "duration_seconds": 58.68544600938967
    }
  ],
  "results": {
    "total_tokens": {
      "unit": "tokens",
      "value": 3000500,
      "derivation": "successful_requests × (fixed_input_tokens + fixed_output_tokens)",
      "input_run_ids": [
        "scheduler-11264"
      ],
      "measurement_status": "derived"
    },
    "failed_requests": 0,
    "duration_seconds": {
      "unit": "s",
      "value": 58.68544600938967,
      "measurement_status": "measured"
    },
    "requests_per_second": {
      "unit": "req/s",
      "value": 8.52,
      "measurement_status": "measured"
    },
    "successful_requests": 500,
    "input_tokens_per_second": {
      "unit": "tok/s",
      "value": 51120,
      "measurement_status": "measured"
    },
    "output_tokens_per_second": {
      "unit": "tok/s",
      "value": 8.52,
      "measurement_status": "measured"
    }
  },
  "runtime": {
    "engine": "vllm",
    "version": "0.27.1",
    "parameters": {
      "vllm": {
        "moe_backend": "FLASHINFER_CUTLASS",
        "max_model_len": 6144,
        "attention_backend": "TRITON_ATTN",
        "enable_prefix_caching": false,
        "future_scheduler_knob": "preserved",
        "gpu_memory_utilization": 0.95,
        "max_num_batched_tokens": 11264
      }
    },
    "parallelism": {
      "data": 1,
      "tensor": 1,
      "pipeline": 1,
      "replicas": 1
    },
    "command_sanitized": "vllm serve MODEL --max-model-len 6144 --max-num-batched-tokens 11264 --gpu-memory-utilization 0.95 --kv-cache-dtype fp8_e4m3"
  },
  "subject": {
    "type": "model",
    "source": "huggingface",
    "revision": "20df0542b1a86ce19f495ac2eca2c7c12bce82f9",
    "identifier": "unsloth/gemma-4-26B-A4B-it-NVFP4",
    "modalities": [
      "text"
    ],
    "architecture": "Gemma4ForConditionalGeneration",
    "quantization": {
      "scope": "all_linears",
      "format": "compressed-tensors",
      "method": "nvfp4",
      "provider": "unsloth",
      "weight_dtype": "nvfp4",
      "kv_cache_dtype": "fp8_e4m3",
      "activation_dtype": "bfloat16"
    },
    "revision_kind": "commit",
    "parameter_count_total": 26000000000,
    "parameter_count_active": 4000000000
  },
  "ended_at": "2026-08-15T16:00:58.685Z",
  "relation": {
    "type": "sweep_point",
    "group": "max_num_batched_tokens"
  },
  "warnings": [],
  "workload": {
    "mode": "online",
    "task": "text_generation",
    "client": {
      "placement": "same-host",
      "overhead_included": true
    },
    "dataset": {
      "license": "CC0-1.0",
      "generator": "vllm-random",
      "synthetic": true
    },
    "sampling": {
      "temperature": 0,
      "max_output_tokens": 1
    },
    "stopping": {
      "exact_output_tokens": true
    },
    "streaming": false,
    "tokenizer": {
      "revision": "20df0542b1a86ce19f495ac2eca2c7c12bce82f9",
      "identifier": "unsloth/gemma-4-26B-A4B-it-NVFP4"
    },
    "concurrency": 8,
    "random_seed": 42,
    "input_tokens": {
      "kind": "fixed",
      "value": 6000
    },
    "endpoint_type": "openai-compatible-chat",
    "output_tokens": {
      "kind": "fixed",
      "value": 1
    },
    "request_count": 500,
    "arrival_process": "closed_loop",
    "prefix_behavior": {
      "cache_state": "cold",
      "reset_policy": "server_restart",
      "cache_enabled": false,
      "repetition_ratio": 0,
      "shared_prefix_tokens": 0
    },
    "warmup_request_count": 8,
    "request_rate_per_second": null
  },
  "artifacts": [
    {
      "kind": "benchmark_result_json",
      "sha256": "9ca84451584837616509201c58a0b1b9ee9ed44fec11a84c4ec6629f3cb568c4",
      "license": "CC-BY-4.0",
      "filename": "scheduler-11264.json",
      "media_type": "application/json",
      "size_bytes": 18432,
      "visibility": "public",
      "run_client_id": "scheduler-11264",
      "client_artifact_id": "scheduler-11264-raw",
      "raw_requests_included": false
    }
  ],
  "started_at": "2026-08-15T16:00:00.000Z",
  "environment": {
    "extensions": {
      "runpile.demo.gpu_rotation": 0
    },
    "accelerators": [
      {
        "count": 1,
        "model": "GeForce RTX 5090",
        "driver": "575.64.03",
        "vendor": "NVIDIA",
        "architecture": "Blackwell",
        "device_indices": [
          0
        ],
        "power_limit_watts": 575,
        "memory_bytes_per_device": 34359738368
      }
    ]
  },
  "client_run_id": "scheduler-11264",
  "duration_seconds": 58.68544600938967
}