{ "recorded_on": "2026-10-16", "operator": "Superfastcode team (first-party measurement)", "purpose": "Small fixed-marker compatibility observation; no comparative speed, uptime or coding-quality claim.", "endpoint": "https://api.superfastcode.com", "machine": "Same macOS arm64 host; client location, gateway region and upstream region not independently verified.", "concurrency": 1, "warmups": 0, "fixed_prompt": "Do not use tools or inspect any files. Reply with exactly: superfastcode-benchmark-ok", "method": { "cli_elapsed": "Process launch to exit and marker validation; includes client startup, context, local bridge, TLS/network and inference.", "first_text_delta": "Elapsed from local bridge starting HTTPS forwarding to first nonempty response.output_text.delta (Codex) or content_block_delta/text_delta (Claude). Not first byte, not gateway-only latency.", "completion": "Protocol completion event plus HTTP 200 and exit 0 plus exact final marker.", "billing": "Read-only ledger rows uniquely matched by narrow per-attempt UTC window, model, input/output and cached-token counts. Edge x-request-id differs from ledger request_id; no direct ID match claimed. Aggregate wallet deltas were rejected because of concurrent traffic.", "test_account_multiplier": "0.01 = ledger actual_cost / ledger total_cost for every matched row. This is a test-account condition, not the public customer tariff. USD-denominated account-credit units, not a cash purchase or provider invoice.", "client_configuration": "Codex: ignore user config/rules, ephemeral, read-only, shell feature disabled, low effort, zero configured transport retries, HTTP Responses. Claude: bare, no setting sources, strict empty MCP, no tools/skills/session persistence, low effort, thinking budget zero. No fallback configured.", "context_limit": "Temporary empty working directory; no project files/tools used. Codex still includes ambient client/global instruction metadata and has ~20.6k input tokens including cache. Claude bare mode has 139\u2013141 uncached input tokens. Contexts and models differ: do not compare their latency or cost as a gateway-only effect.", "sample_limit": "Five per client in one short window, default cache state, no controlled cache flush; insufficient for p95, uptime or SLA estimates.", "harness_limit": "The existing downloadable gateway-benchmark.py measures CLI elapsed only. These first-text timings require the additional local protocol observer used for this observation." }, "summary": [ { "client": "codex", "attempts": 5, "marker_successes": 5, "http_requests": 5, "median_cli_elapsed_ms": 7585.891, "median_transport_first_text_delta_ms": 3277.061, "ledger_base_cost_usd_units": "0.1678500000", "ledger_deducted_usd_credit_units": "0.0016785000" }, { "client": "claude", "attempts": 5, "marker_successes": 5, "http_requests": 5, "median_cli_elapsed_ms": 1946.729, "median_transport_first_text_delta_ms": 1290.255, "ledger_base_cost_usd_units": "0.0021480000", "ledger_deducted_usd_credit_units": "0.0000214800" } ], "samples": [ { "client": "codex", "attempt": 1, "client_version": "codex-cli 0.161.0", "model_requested": "gpt-6.1-sol", "model_observed": "gpt-6.1-sol", "started_at_utc": "2026-10-16T07:06:59.965338+00:00", "cli_elapsed_ms": 9349.715, "transport_first_text_delta_ms": 3664.025, "transport_elapsed_ms": 3833.538, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 16521, "cache_read_tokens": 4096, "cache_creation_tokens": 0, "output_tokens": 10, "ledger_base_cost_usd_units": "0.0335516000", "ledger_deducted_usd_credit_units": "0.0003355160" }, { "client": "codex", "attempt": 2, "client_version": "codex-cli 0.161.0", "model_requested": "gpt-6.1-sol", "model_observed": "gpt-6.1-sol", "started_at_utc": "2026-10-16T07:08:01.571421+00:00", "cli_elapsed_ms": 7547.53, "transport_first_text_delta_ms": 3034.29, "transport_elapsed_ms": 3252.442, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 16519, "cache_read_tokens": 4096, "cache_creation_tokens": 0, "output_tokens": 10, "ledger_base_cost_usd_units": "0.0335476000", "ledger_deducted_usd_credit_units": "0.0003354760" }, { "client": "codex", "attempt": 3, "client_version": "codex-cli 0.161.0", "model_requested": "gpt-6.1-sol", "model_observed": "gpt-6.1-sol", "started_at_utc": "2026-10-16T07:08:10.997966+00:00", "cli_elapsed_ms": 7399.969, "transport_first_text_delta_ms": 3764.598, "transport_elapsed_ms": 3911.87, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 16519, "cache_read_tokens": 4096, "cache_creation_tokens": 0, "output_tokens": 10, "ledger_base_cost_usd_units": "0.0335476000", "ledger_deducted_usd_credit_units": "0.0003354760" }, { "client": "codex", "attempt": 4, "client_version": "codex-cli 0.161.0", "model_requested": "gpt-6.1-sol", "model_observed": "gpt-6.1-sol", "started_at_utc": "2026-10-16T07:08:21.218991+00:00", "cli_elapsed_ms": 7656.846, "transport_first_text_delta_ms": 2960.097, "transport_elapsed_ms": 3016.575, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 16519, "cache_read_tokens": 4096, "cache_creation_tokens": 0, "output_tokens": 10, "ledger_base_cost_usd_units": "0.0335476000", "ledger_deducted_usd_credit_units": "0.0003354760" }, { "client": "codex", "attempt": 5, "client_version": "codex-cli 0.161.0", "model_requested": "gpt-6.1-sol", "model_observed": "gpt-6.1-sol", "started_at_utc": "2026-10-16T07:08:29.604784+00:00", "cli_elapsed_ms": 7585.891, "transport_first_text_delta_ms": 3277.061, "transport_elapsed_ms": 3391.363, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 16573, "cache_read_tokens": 4096, "cache_creation_tokens": 0, "output_tokens": 10, "ledger_base_cost_usd_units": "0.0336556000", "ledger_deducted_usd_credit_units": "0.0003365560" }, { "client": "claude", "attempt": 1, "client_version": "2.1.300 (Claude Code)", "model_requested": "claude-sonnet-5-5", "model_observed": "claude-sonnet-5-5", "started_at_utc": "2026-10-16T07:07:24.345035+00:00", "cli_elapsed_ms": 2111.798, "transport_first_text_delta_ms": 1366.374, "transport_elapsed_ms": 1830.033, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 139, "cache_read_tokens": 0, "cache_creation_tokens": 0, "output_tokens": 15, "ledger_base_cost_usd_units": "0.0004280000", "ledger_deducted_usd_credit_units": "0.0000042800" }, { "client": "claude", "attempt": 2, "client_version": "2.1.300 (Claude Code)", "model_requested": "claude-sonnet-5-5", "model_observed": "claude-sonnet-5-5", "started_at_utc": "2026-10-16T07:09:04.399986+00:00", "cli_elapsed_ms": 2123.807, "transport_first_text_delta_ms": 1454.678, "transport_elapsed_ms": 1962.167, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 141, "cache_read_tokens": 0, "cache_creation_tokens": 0, "output_tokens": 15, "ledger_base_cost_usd_units": "0.0004320000", "ledger_deducted_usd_credit_units": "0.0000043200" }, { "client": "claude", "attempt": 3, "client_version": "2.1.300 (Claude Code)", "model_requested": "claude-sonnet-5-5", "model_observed": "claude-sonnet-5-5", "started_at_utc": "2026-10-16T07:09:08.548556+00:00", "cli_elapsed_ms": 1750.907, "transport_first_text_delta_ms": 1131.417, "transport_elapsed_ms": 1568.518, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 140, "cache_read_tokens": 0, "cache_creation_tokens": 0, "output_tokens": 15, "ledger_base_cost_usd_units": "0.0004300000", "ledger_deducted_usd_credit_units": "0.0000043000" }, { "client": "claude", "attempt": 4, "client_version": "2.1.300 (Claude Code)", "model_requested": "claude-sonnet-5-5", "model_observed": "claude-sonnet-5-5", "started_at_utc": "2026-10-16T07:09:12.328741+00:00", "cli_elapsed_ms": 1849.636, "transport_first_text_delta_ms": 1195.114, "transport_elapsed_ms": 1647.293, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 139, "cache_read_tokens": 0, "cache_creation_tokens": 0, "output_tokens": 15, "ledger_base_cost_usd_units": "0.0004280000", "ledger_deducted_usd_credit_units": "0.0000042800" }, { "client": "claude", "attempt": 5, "client_version": "2.1.300 (Claude Code)", "model_requested": "claude-sonnet-5-5", "model_observed": "claude-sonnet-5-5", "started_at_utc": "2026-10-16T07:09:16.175892+00:00", "cli_elapsed_ms": 1946.729, "transport_first_text_delta_ms": 1290.255, "transport_elapsed_ms": 1759.381, "http_status": 200, "protocol_completed": true, "marker_success": true, "input_tokens_excluding_cache": 140, "cache_read_tokens": 0, "cache_creation_tokens": 0, "output_tokens": 15, "ledger_base_cost_usd_units": "0.0004300000", "ledger_deducted_usd_credit_units": "0.0000043000" } ] }