{ "schemaVersion": 1, "source": { "origin": "live", "url": "https://benchmarks.speko.ai/data.json", "sha256": "6afea0390756aca3386a2555ccdb833064c4c75532f607b8d13e7b2d26ff7569", "platformCommit": "99972fbe223609c8658abe38e00a7d8630fa37ef" }, "boards": { "stt": { "id": "stt", "title": "Speech-to-Text", "navLabel": "Speech-to-Text", "subtitle": "Ranked by end-of-turn latency for a live agent.", "meta": "English · read speech (FLEURS, n=50) · batch 2026-07-03, streaming 2026-07-21", "method": "WER normalization, identical on reference and hypothesis: NFC, lowercased, hyphens and slashes split into words, apostrophes deleted, remaining punctuation and symbols dropped, spelled-out English numbers and digit ordinals folded to digits. Currency, percent and spoken decimals have no rule and still score as errors - the inverse-text-normalization flag below.", "measured": "2026-07-03", "measuredThrough": null, "laterRuns": [], "rankCol": "endpoint", "headline": [ "endpoint", "wer", "cost" ], "scatter": { "x": "endpoint", "y": "wer" }, "columns": [ { "key": "wer", "label": "WER · batch", "dir": "lower", "info": "Word Error Rate on FLEURS read English (n=50), batch API. Read speech, not studio-clean audio. Lower is better." }, { "key": "werStream", "label": "WER · stream", "dir": "lower", "info": "WER on the same clips through the live streaming socket — the path a voice agent runs on. Lower is better." }, { "key": "endpoint", "label": "Finalize", "dir": "lower", "info": "End-of-speech to final transcript, p50 (us-east4, n=30). Whisker spans p50→p90." }, { "key": "ttft", "label": "Time to first token", "dir": "lower", "info": "First partial word, p50 (us-east4). Whisker spans p50→p90." }, { "key": "cost", "label": "Cost / min", "dir": "lower", "info": "$ per minute of audio, vendor list, streaming path. Pay-as-you-go where offered, otherwise the cheapest paid plan. Lower = cheaper; \"—\" where a vendor publishes no per-minute rate." } ], "rows": [ { "provider": "Smallest AI", "model": "Pulse", "slug": "smallest-ai-pulse", "stack": null, "pin": "smallest:pulse", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": "WER inflated by inverse-text-normalization — spelled-out numbers score as errors.", "cells": { "wer": { "v": "5.1%", "s": 5.1, "tone": "warn", "dim": false, "ci": null, "note": null }, "werStream": { "v": "8.0%", "s": 8, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "178ms", "s": 178, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.39s", "s": 1394, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "~$0.0050", "s": 0.005, "tone": null, "dim": false, "ci": null, "note": "Streaming Pulse. The vendor never publishes an exact rate, and its own pricing page contradicts this model page with ~$0.009/min." } } }, { "provider": "Deepgram", "model": "Nova-3", "slug": "deepgram-nova-3", "stack": null, "pin": "deepgram:nova-3", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "wer": { "v": "9.8%", "s": 9.8, "tone": "warn", "dim": false, "ci": null, "note": null }, "werStream": { "v": "12.9%", "s": 12.9, "tone": "bad", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "106ms", "s": 106, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.12s", "s": 1124, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0048", "s": 0.0048, "tone": null, "dim": false, "ci": null, "note": "Streaming pay-as-you-go. Vendor flags it as a limited-time promotional rate; pre-recorded is $0.0043." } } }, { "provider": "Deepgram", "model": "Flux", "slug": "deepgram-flux", "stack": null, "pin": "deepgram:flux-general-en", "pinByMetric": null, "lead": false, "status": { "label": "GOOD", "tone": "good" }, "measured": null, "flag": "Finalize includes Flux’s own turn decision — no forced finalize, no batch endpoint.", "cells": { "wer": { "v": "—", "s": null, "tone": null, "dim": false, "ci": null, "note": null }, "werStream": { "v": "6.6%", "s": 6.6, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "406ms", "s": 406, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4 · n=30 · 2026-08-04", "note": null }, "ttft": { "v": "1.11s", "s": 1109, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · 2026-08-04", "note": null }, "cost": { "v": "$0.0065", "s": 0.0065, "tone": null, "dim": false, "ci": null, "note": "Flux English streaming pay-as-you-go. Vendor flags it as a limited-time promotional rate; flux-general-multi is $0.0078." } } }, { "provider": "Cartesia", "model": "Ink-2", "slug": "cartesia-ink-2", "stack": null, "pin": "cartesia:ink-2", "pinByMetric": null, "lead": false, "status": { "label": "WEAK", "tone": "bad" }, "measured": null, "flag": "Batch WER inflated by truncation; the streaming path it runs on scores better.", "cells": { "wer": { "v": "11.0%", "s": 11, "tone": "bad", "dim": false, "ci": null, "note": null }, "werStream": { "v": "9.9%", "s": 9.9, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "102ms", "s": 102, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.16s", "s": 1161, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0090", "s": 0.009, "tone": null, "dim": false, "ci": null, "note": "Ink-2 bills 3 credits/sec on Pro, the cheapest paid plan; $0.0071 at Startup, $0.0067 at Scale. The 1 credit/sec rate is Ink-Whisper, a different model." } } }, { "provider": "Alibaba", "model": "Qwen3-ASR", "slug": "alibaba-qwen3-asr", "stack": null, "pin": "alibaba:qwen3-asr-flash", "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": "Asia-hosted endpoint — p50 carries the network round-trip to Singapore.", "cells": { "wer": { "v": "2.8%", "s": 2.8, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "4.0%", "s": 4, "tone": "good", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "424ms", "s": 424, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "6.60s", "s": 6596, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0054", "s": 0.0054, "tone": null, "dim": false, "ci": null, "note": "qwen3-asr-flash-realtime, Singapore. The file/sync SKU is $0.0021." } } }, { "provider": "Google", "model": "Chirp 3", "slug": "google-chirp-3", "stack": null, "pin": "google:chirp_3", "pinByMetric": null, "lead": false, "status": { "label": "GOOD", "tone": "good" }, "measured": null, "flag": null, "cells": { "wer": { "v": "3.9%", "s": 3.9, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "7.4%", "s": 7.4, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "581ms", "s": 581, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "5.60s", "s": 5595, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0160", "s": 0.016, "tone": "bad", "dim": false, "ci": null, "note": "Speech-to-Text V2 Standard, first tier — one rate for streaming, sync and batch. Dynamic Batch Recognition is $0.003." } } }, { "provider": "ElevenLabs", "model": "Scribe v2", "slug": "elevenlabs-scribe-v2", "stack": null, "pin": "elevenlabs:scribe_v2", "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": null, "cells": { "wer": { "v": "3.1%", "s": 3.1, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "3.4%", "s": 3.4, "tone": "good", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "233ms", "s": 233, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "2.18s", "s": 2184, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0065", "s": 0.0065, "tone": null, "dim": false, "ci": null, "note": "Scribe v2 Realtime at $0.39/hr, flat across every tier. Batch Scribe v2 is $0.0037." } } }, { "provider": "Inworld", "model": "Realtime STT-1", "slug": "inworld-realtime-stt-1", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": "Finalizes eagerly — ~1 in 5 turns commit before speech ends.", "cells": { "wer": { "v": "3.3%", "s": 3.3, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "3.6%", "s": 3.6, "tone": "good", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "139ms", "s": 139, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=50", "note": null }, "ttft": { "v": "1.21s", "s": 1209, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0025", "s": 0.0025, "tone": "good", "dim": false, "ci": null, "note": "On-demand $0.15/hr. The $0.10/hr headline needs a paid Creator subscription." } } }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "slug": "assemblyai-universal-3-5-pro", "stack": null, "pin": "assemblyai:universal-3-5-pro", "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": null, "cells": { "wer": { "v": "2.0%", "s": 2, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "2.0%", "s": 2, "tone": "good", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "66ms", "s": 66, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.32s", "s": 1324, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0075", "s": 0.0075, "tone": null, "dim": false, "ci": null, "note": "Universal-3.5 Pro Realtime at $0.45/hr. Async pre-recorded is $0.0035. Diarization and other add-ons bill on top." } } }, { "provider": "Soniox", "model": "stt-rt-v5", "slug": "soniox-stt-rt-v5", "stack": null, "pin": "soniox:stt-rt-v5", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "wer": { "v": "7.5%", "s": 7.5, "tone": "warn", "dim": false, "ci": "95% CI 4.8–10.7 · n=50", "note": null }, "werStream": { "v": "7.3%", "s": 7.3, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "78ms", "s": 78, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.04s", "s": 1039, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0020", "s": 0.002, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "slug": "openai-gpt-4o-transcribe", "stack": null, "pin": "openai:gpt-4o-transcribe", "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": null, "cells": { "wer": { "v": "2.3%", "s": 2.3, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "5.8%", "s": 5.8, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "572ms", "s": 572, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "5.06s", "s": 5059, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0060", "s": 0.006, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "GPT Transcribe / GPT Live Transcribe", "slug": "openai-gpt-transcribe-gpt-live-transcribe", "stack": null, "pin": null, "pinByMetric": { "wer": "openai:gpt-transcribe", "werStream": "openai:gpt-live-transcribe", "endpoint": "openai:gpt-live-transcribe", "ttft": "openai:gpt-live-transcribe", "cost": "openai:gpt-transcribe" }, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": "Two models: batch WER is gpt-transcribe, streaming is gpt-live-transcribe — not interchangeable.", "cells": { "wer": { "v": "2.5%", "s": 2.5, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "4.5%", "s": 4.5, "tone": "good", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "1124ms", "s": 1124, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.77s", "s": 1769, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0170", "s": 0.017, "tone": "bad", "dim": false, "ci": null, "note": "gpt-live-transcribe, the streaming path this row is recommended for. Batch gpt-transcribe is $0.0045/min." } } }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "slug": "openai-gpt-4o-mini-transcribe", "stack": null, "pin": "openai:gpt-4o-mini-transcribe", "pinByMetric": null, "lead": false, "status": { "label": "EXCELLENT", "tone": "good" }, "measured": null, "flag": null, "cells": { "wer": { "v": "2.7%", "s": 2.7, "tone": "good", "dim": false, "ci": null, "note": null }, "werStream": { "v": "6.4%", "s": 6.4, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "460ms", "s": 460, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "4.98s", "s": 4978, "tone": "bad", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0030", "s": 0.003, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "xAI", "model": "Grok STT", "slug": "xai-grok-stt", "stack": null, "pin": "xai:stt", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "wer": { "v": "4.8%", "s": 4.8, "tone": "warn", "dim": false, "ci": null, "note": "Loudness-normalized best-case; degrades on raw input." }, "werStream": { "v": "10.9%", "s": 10.9, "tone": "warn", "dim": false, "ci": null, "note": "Raw audio, n=31: 15/50 very-quiet FLEURS clips excluded where the model returns no transcript (loudness-sensitive; batch 4.8% is loudness-normalized)." }, "endpoint": { "v": "305ms", "s": 305, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.34s", "s": 1341, "tone": "good", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0033", "s": 0.0033, "tone": null, "dim": false, "ci": null, "note": "grok-stt streaming at $0.20/hr. REST is $0.0017." } } }, { "provider": "Gradium", "model": "Gradium ASR", "slug": "gradium-gradium-asr", "stack": null, "pin": "gradium:default", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "wer": { "v": "8.4%", "s": 8.4, "tone": "warn", "dim": false, "ci": null, "note": null }, "werStream": { "v": "11.7%", "s": 11.7, "tone": "warn", "dim": false, "ci": null, "note": null }, "endpoint": { "v": "334ms", "s": 334, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4 · n=30", "note": null }, "ttft": { "v": "1.83s", "s": 1830, "tone": "warn", "dim": false, "ci": "p50–p90 · us-east4", "note": null }, "cost": { "v": "$0.0104", "s": 0.0104, "tone": null, "dim": false, "ci": null, "note": "3 credits/sec on the XS tier, the cheapest paid plan; $0.0068 at the L tier." } } } ] }, "stt-codeswitch": { "id": "stt-codeswitch", "title": "Code-switching", "navLabel": "Code-switching", "subtitle": "Two languages in one utterance — error rate and whether the second language survives, per pair, over the live stream.", "meta": null, "method": null, "measured": null, "measuredThrough": null, "laterRuns": [], "rankCol": "pairs", "headline": [ "pairs", "zh", "es" ], "scatter": null, "columns": [ { "key": "pairs", "label": "Pairs handled", "dir": "higher", "info": "Language pairs where the second language survives (both-language coverage over 50%), of 4 tested." }, { "key": "zh", "label": "EN ↔ ZH", "dir": "lower", "info": "Mixed-token error rate on Mandarin-English code-switch (ASCEND, streaming); caption = both-language coverage. Lower is better." }, { "key": "es", "label": "EN ↔ ES", "dir": "lower", "info": "Word error rate on English-Spanish code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." }, { "key": "de", "label": "EN ↔ DE", "dir": "lower", "info": "Word error rate on English-German code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." }, { "key": "fr", "label": "EN ↔ FR", "dir": "lower", "info": "Word error rate on English-French code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." } ], "rows": [ { "provider": "Soniox", "model": "stt-rt-v5", "slug": "soniox-stt-rt-v5", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "pairs": { "v": "4 / 4", "s": 4, "tone": "good", "dim": false, "ci": null, "note": null }, "zh": { "v": "11.8%", "s": 11.8, "tone": "good", "dim": false, "ci": "88% both-lang", "note": null }, "es": { "v": "4.9%", "s": 4.9, "tone": "good", "dim": false, "ci": "97% both-lang", "note": null }, "de": { "v": "5.1%", "s": 5.1, "tone": "good", "dim": false, "ci": "100% both-lang", "note": null }, "fr": { "v": "8.8%", "s": 8.8, "tone": "good", "dim": false, "ci": "89% both-lang", "note": null } } }, { "provider": "AssemblyAI", "model": "universal-3-5-pro", "slug": "assemblyai-universal-3-5-pro", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "pairs": { "v": "4 / 4", "s": 4, "tone": "good", "dim": false, "ci": null, "note": null }, "zh": { "v": "9.8%", "s": 9.8, "tone": "good", "dim": false, "ci": "100% both-lang", "note": null }, "es": { "v": "3.5%", "s": 3.5, "tone": "good", "dim": false, "ci": "100% both-lang", "note": null }, "de": { "v": "3.7%", "s": 3.7, "tone": "good", "dim": false, "ci": "98% both-lang", "note": null }, "fr": { "v": "2.6%", "s": 2.6, "tone": "good", "dim": false, "ci": "93% both-lang", "note": null } } }, { "provider": "ElevenLabs", "model": "scribe_v2_realtime", "slug": "elevenlabs-scribe-v2-realtime", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Strong on Latin-script pairs; anglicizes Mandarin (drops it to English).", "cells": { "pairs": { "v": "3 / 4", "s": 3, "tone": "warn", "dim": false, "ci": null, "note": null }, "zh": { "v": "75.0%", "s": 75, "tone": "bad", "dim": false, "ci": "34% both-lang", "note": null }, "es": { "v": "3.8%", "s": 3.8, "tone": "good", "dim": false, "ci": "100% both-lang", "note": null }, "de": { "v": "3.1%", "s": 3.1, "tone": "good", "dim": false, "ci": "98% both-lang", "note": null }, "fr": { "v": "7.8%", "s": 7.8, "tone": "good", "dim": false, "ci": "89% both-lang", "note": null } } } ] }, "tts": { "id": "tts", "title": "Text-to-Speech", "navLabel": "Text-to-Speech", "subtitle": "Ranked by how human it sounds", "meta": "Naturalness and win % / en-US, blind human panel, no brand labels / 15 systems / bootstrap 95% CI / panel July 2026", "method": null, "measured": "2026-07-03", "measuredThrough": null, "laterRuns": [], "rankCol": "natural", "headline": [ "natural", "synth", "cost" ], "scatter": { "x": "synth", "y": "natural" }, "columns": [ { "key": "natural", "label": "Naturalness", "dir": "higher", "info": "Blind human A/B preference as an Internal MOS (1–5); higher = preferred." }, { "key": "ci", "label": "Win % · 95% CI", "dir": "higher", "info": "Head-to-head win rate against the rest of the field, with its 95% confidence interval. Overlapping intervals = statistical tie — read the interval, not just the point. Higher = preferred." }, { "key": "drift", "label": "Drift", "dir": "lower", "info": "Voice steadiness: mean pairwise embedding distance over 20 repeats. Lower = steadier. \"—\" = not yet run for this model." }, { "key": "synth", "label": "Synth p50", "dir": "lower", "info": "Time to first audio, p50 (n=30, us-east4, sequential). Lower is better." }, { "key": "cost", "label": "Cost / 1M chars", "dir": "lower", "info": "$ per 1M characters, vendor list. Pay-as-you-go where offered, otherwise the cheapest paid plan. Lower = cheaper; \"~\" = converted from token or hourly pricing at 900 chars/min of speech." } ], "rows": [ { "provider": "ElevenLabs", "model": "eleven_v3", "slug": "elevenlabs-eleven-v3", "stack": null, "pin": "elevenlabs:eleven_v3", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.51", "s": 3.51, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "66%", "s": 66, "tone": "good", "dim": false, "ci": "60–72%", "note": null }, "drift": { "v": "20", "s": 20, "tone": null, "dim": false, "ci": null, "note": null }, "synth": { "v": "481ms", "s": 481, "tone": null, "dim": false, "ci": "481–559 p50–p90", "note": null }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "slug": "gemini-gemini-3-1-flash-tts-preview", "stack": null, "pin": "google-tts:gemini-3.1-flash-tts-preview", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.51", "s": 3.51, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "66%", "s": 66, "tone": "good", "dim": false, "ci": "59–72%", "note": null }, "drift": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": null }, "synth": { "v": "978ms", "s": 978, "tone": null, "dim": false, "ci": "978–1235 p50–p90", "note": null }, "cost": { "v": "~$33.3", "s": 33.3, "tone": null, "dim": false, "ci": null, "note": "Token-billed: $1/1M text in, $20/1M audio out; Google states 25 audio tokens/sec (ai.google.dev/gemini-api/docs/pricing). At 900 chars/min of speech that is ~$33 per 1M chars; text-in adds ~$0.25, negligible." } } }, { "provider": "Deepgram", "model": "aura-2", "slug": "deepgram-aura-2", "stack": null, "pin": "deepgram:aura-2", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.47", "s": 3.47, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "65%", "s": 65, "tone": "good", "dim": false, "ci": "58–70%", "note": null }, "drift": { "v": "9", "s": 9, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "125ms", "s": 125, "tone": "good", "dim": false, "ci": "125–138 p50–p90", "note": null }, "cost": { "v": "$30.0", "s": 30, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "Cartesia", "model": "sonic-3.5", "slug": "cartesia-sonic-3-5", "stack": null, "pin": "cartesia:sonic-3.5", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.42", "s": 3.42, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "65%", "s": 65, "tone": "good", "dim": false, "ci": "58–71%", "note": null }, "drift": { "v": "13", "s": 13, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "121ms", "s": 121, "tone": "good", "dim": false, "ci": "121–236 p50–p90", "note": null }, "cost": { "v": "$50.0", "s": 50, "tone": null, "dim": false, "ci": null, "note": "1 credit/char on Pro, the cheapest paid plan; $39.2 at Startup, $37.4 at Scale. Cloned voices bill 1.5x." } } }, { "provider": "Soniox", "model": "tts-rt-v1", "slug": "soniox-tts-rt-v1", "stack": null, "pin": "soniox:tts-rt-v1", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.39", "s": 3.39, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "63%", "s": 63, "tone": "good", "dim": false, "ci": "56–69%", "note": null }, "drift": { "v": "24", "s": 24, "tone": "warn", "dim": false, "ci": null, "note": null }, "synth": { "v": "362ms", "s": 362, "tone": "good", "dim": false, "ci": "362–388 p50–p90", "note": null }, "cost": { "v": "~$13.0", "s": 13, "tone": "good", "dim": false, "ci": null, "note": "Token-billed: $4/1M text in, $21.50/1M audio out; the vendor states ~$0.70 per hour of generated speech (soniox.com/pricing). At 900 chars/min (54,000 chars/hour) that is ~$13 per 1M chars." } } }, { "provider": "Speechify", "model": "simba-3.2", "slug": "speechify-simba-3-2", "stack": null, "pin": "speechify:simba-3.2", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.41", "s": 3.41, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "63%", "s": 63, "tone": "good", "dim": false, "ci": "57–69%", "note": null }, "drift": { "v": "7", "s": 7, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "345ms", "s": 345, "tone": "good", "dim": false, "ci": "345–375 p50–p90", "note": null }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "dim": false, "ci": null, "note": "Vendor list (Starter tier); $6–8 per 1M at Pro/Scale volume." } } }, { "provider": "Inworld", "model": "inworld-tts-2", "slug": "inworld-inworld-tts-2", "stack": null, "pin": "inworld:inworld-tts-2", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.34", "s": 3.34, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "63%", "s": 63, "tone": "good", "dim": false, "ci": "57–69%", "note": null }, "drift": { "v": "18", "s": 18, "tone": null, "dim": false, "ci": null, "note": null }, "synth": { "v": "116ms", "s": 116, "tone": "good", "dim": false, "ci": "116–125 p50–p90", "note": null }, "cost": { "v": "$25.0", "s": 25, "tone": "good", "dim": false, "ci": null, "note": "On-demand. Drops to $20 on Creator and $12.50 on Growth." } } }, { "provider": "Fish Audio", "model": "s2.1-pro", "slug": "fish-audio-s2-1-pro", "stack": null, "pin": "fishaudio:s2.1-pro", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "~3.35", "s": 3.35, "tone": null, "dim": true, "ci": null, "note": null }, "ci": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": null }, "drift": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": null }, "synth": { "v": "185ms", "s": 185, "tone": "good", "dim": false, "ci": "185–235 p50–p90", "note": null }, "cost": { "v": "$15.0", "s": 15, "tone": "good", "dim": false, "ci": null, "note": "Billed per UTF-8 byte, so this holds for Latin text only — CJK runs ~3x. The s2.1-pro-free tier is $0 through 2026-08-31." } } }, { "provider": "Smallest", "model": "lightning_v3.1", "slug": "smallest-lightning-v3-1", "stack": null, "pin": "smallest:lightning_v3.1", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "3.25", "s": 3.25, "tone": "good", "dim": false, "ci": null, "note": null }, "ci": { "v": "58%", "s": 58, "tone": "good", "dim": false, "ci": "51–64%", "note": null }, "drift": { "v": "0.27", "s": 0.27, "tone": "good", "dim": false, "ci": "deterministic", "note": null }, "synth": { "v": "173ms", "s": 173, "tone": "good", "dim": false, "ci": "173–193 p50–p90", "note": null }, "cost": { "v": "$25.0", "s": 25, "tone": "good", "dim": false, "ci": null, "note": "Pay-as-you-go (Pro tier); ~$0.25/10k chars." } } }, { "provider": "xAI Grok", "model": "grok-tts", "slug": "xai-grok-grok-tts", "stack": null, "pin": "xai:tts", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.96", "s": 2.96, "tone": null, "dim": false, "ci": null, "note": null }, "ci": { "v": "48%", "s": 48, "tone": null, "dim": false, "ci": "42–55%", "note": null }, "drift": { "v": "15", "s": 15, "tone": null, "dim": false, "ci": null, "note": null }, "synth": { "v": "272ms", "s": 272, "tone": "good", "dim": false, "ci": "272–325 p50–p90", "note": null }, "cost": { "v": "$15.0", "s": 15, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "Gradium", "model": "default", "slug": "gradium-default", "stack": null, "pin": "gradium:default", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.69", "s": 2.69, "tone": null, "dim": false, "ci": null, "note": null }, "ci": { "v": "38%", "s": 38, "tone": null, "dim": false, "ci": "32–45%", "note": null }, "drift": { "v": "12", "s": 12, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "244ms", "s": 244, "tone": "good", "dim": false, "ci": "244–278 p50–p90", "note": null }, "cost": { "v": "$57.8", "s": 57.8, "tone": null, "dim": false, "ci": null, "note": "1 credit/char on the XS tier, the cheapest paid plan; $35.9 at the L tier." } } }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "slug": "openai-gpt-4o-mini-tts", "stack": null, "pin": "openai:gpt-4o-mini-tts", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.56", "s": 2.56, "tone": null, "dim": false, "ci": null, "note": null }, "ci": { "v": "37%", "s": 37, "tone": null, "dim": false, "ci": "31–43%", "note": null }, "drift": { "v": "22", "s": 22, "tone": "warn", "dim": false, "ci": null, "note": null }, "synth": { "v": "691ms", "s": 691, "tone": null, "dim": false, "ci": "691–1171 p50–p90", "note": null }, "cost": { "v": "~$20.0", "s": 20, "tone": "good", "dim": false, "ci": null, "note": "Token-billed: $0.60/1M text in, $12/1M audio out; OpenAI publishes no per-character or per-minute rate. At ~25 audio tokens/sec (not OpenAI-published) and 900 chars/min, ~$20 per 1M chars. Legacy tts-1 is $15, tts-1-hd $30 per 1M chars." } } }, { "provider": "Rime", "model": "arcanav3", "slug": "rime-arcanav3", "stack": null, "pin": "rime:arcanav3", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.59", "s": 2.59, "tone": null, "dim": false, "ci": null, "note": null }, "ci": { "v": "36%", "s": 36, "tone": null, "dim": false, "ci": "31–42%", "note": null }, "drift": { "v": "21", "s": 21, "tone": null, "dim": false, "ci": null, "note": null }, "synth": { "v": "238ms", "s": 238, "tone": "good", "dim": false, "ci": "238–248 p50–p90", "note": null }, "cost": { "v": "$40.0", "s": 40, "tone": null, "dim": false, "ci": null, "note": "Arcana on Starter; $30 on Growth. The vendor pricing page contradicts itself — its FAQ quotes $0.05/1k and its card quotes the cheaper Mist rate." } } }, { "provider": "MiniMax", "model": "speech-2.8-hd", "slug": "minimax-speech-2-8-hd", "stack": null, "pin": "minimax:speech-2.8-hd", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.60", "s": 2.6, "tone": null, "dim": false, "ci": null, "note": null }, "ci": { "v": "36%", "s": 36, "tone": null, "dim": false, "ci": "30–43%", "note": null }, "drift": { "v": "12", "s": 12, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "294ms", "s": 294, "tone": "good", "dim": false, "ci": "294–354 p50–p90", "note": null }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "dim": false, "ci": null, "note": "HD tier; speech-2.8-turbo ≈ $60/1M." } } }, { "provider": "Hume", "model": "octave-2", "slug": "hume-octave-2", "stack": null, "pin": "hume:octave-2", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "2.30", "s": 2.3, "tone": "warn", "dim": false, "ci": null, "note": null }, "ci": { "v": "29%", "s": 29, "tone": "warn", "dim": false, "ci": "24–34%", "note": null }, "drift": { "v": "25", "s": 25, "tone": "warn", "dim": false, "ci": null, "note": null }, "synth": { "v": "448ms", "s": 448, "tone": null, "dim": false, "ci": "448–497 p50–p90", "note": null }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "dim": false, "ci": null, "note": "Starter tier, no pay-as-you-go rate published. Falls to $50 on Business; entry overage is $150." } } }, { "provider": "Qwen", "model": "qwen3-tts-flash", "slug": "qwen-qwen3-tts-flash", "stack": null, "pin": "alibaba:qwen3-tts-flash", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "1.98", "s": 1.98, "tone": "bad", "dim": false, "ci": null, "note": null }, "ci": { "v": "21%", "s": 21, "tone": "bad", "dim": false, "ci": "16–27%", "note": null }, "drift": { "v": "21", "s": 21, "tone": null, "dim": false, "ci": null, "note": null }, "synth": { "v": "472ms", "s": 472, "tone": null, "dim": false, "ci": "472–504 p50–p90", "note": null }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "dim": false, "ci": null, "note": "International (Singapore) list: $0.10 per 10,000 input characters, output audio not billed (alibabacloud.com/help/en/model-studio/model-pricing). A true per-character rate, no conversion. Mainland-China scope is ~$0.115/10k." } } }, { "provider": "Palabra", "model": "palabra-tts-v1", "slug": "palabra-palabra-tts-v1", "stack": null, "pin": "palabra:palabra-tts-v1", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "natural": { "v": "~3.10", "s": 3.1, "tone": null, "dim": true, "ci": null, "note": null }, "ci": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": null }, "drift": { "v": "11", "s": 11, "tone": "good", "dim": false, "ci": null, "note": null }, "synth": { "v": "72ms", "s": 72, "tone": "good", "dim": false, "ci": "72–79 p50–p90", "note": null }, "cost": { "v": "$30.0", "s": 30, "tone": "good", "dim": false, "ci": null, "note": null } } } ] }, "llm": { "id": "llm", "title": "LLM", "navLabel": "LLM", "subtitle": "Ranked by a speed-weighted score (first-token latency counts most).", "meta": "English · behavior measured 2026-07-17 (n=3 iters) · TTFT/cost from us-east4", "method": null, "measured": "2026-07-17", "measuredThrough": "2026-07-25", "laterRuns": [ { "model": "gpt-5.6-luna", "date": "2026-07-24" }, { "model": "gemini-3.5-flash-lite", "date": "2026-07-25" } ], "rankCol": "score", "headline": [ "score", "task", "ttft" ], "scatter": { "x": "ttft", "y": "task" }, "columns": [ { "key": "score", "label": "Score", "dir": "higher", "info": "Weighted, speed-first: TTFT 50%, task 25%, dead-air 15%, fabrication 10%. Higher is better." }, { "key": "task", "label": "Task done", "dir": "higher", "info": "Multi-step tool tasks completed end-to-end (chains, compound, clarify), % of runs." }, { "key": "fabric", "label": "Fabrication", "dir": "lower", "info": "Invented a policy or fact it was never given, % of out-of-policy questions." }, { "key": "deadair", "label": "Dead-air", "dir": "lower", "info": "Turns where the caller hears silence (no spoken words), % of turns." }, { "key": "toolsilence", "label": "Tool silence", "dir": "lower", "info": "Tool-call turns with no spoken lead-in, % — the caller waits in silence. Not in the score." }, { "key": "ttft", "label": "TTFT p50", "dir": "lower", "info": "Time-to-first-token, p50 (us-east4). Whisker spans p50→p90." }, { "key": "cost", "label": "Cost / 1M tok", "dir": "lower", "info": "$ per 1M output tokens (host / vendor list pricing)." } ], "rows": [ { "provider": "Cerebras", "model": "gemma-4-31b", "slug": "cerebras-gemma-4-31b", "stack": null, "pin": "cerebras:gemma-4-31b", "pinByMetric": null, "lead": true, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": "Speaks some tool calls as text; a repair lifts task 67%→89%.", "cells": { "score": { "v": "92", "s": 92, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "89% †", "s": 89, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "12%", "s": 12, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "36%", "s": 36, "tone": "warn", "dim": false, "ci": null, "note": null }, "ttft": { "v": "192ms", "s": 192, "tone": "good", "dim": false, "ci": "192–232 p50–p90", "note": null }, "cost": { "v": "$1.49", "s": 1.49, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "slug": "anthropic-claude-haiku-4-5", "stack": null, "pin": "anthropic:claude-haiku-4-5", "pinByMetric": null, "lead": false, "status": { "label": "STRONG", "tone": "good" }, "measured": null, "flag": "Perfect on task, fabrication, and dead-air — 532ms TTFT.", "cells": { "score": { "v": "91", "s": 91, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "532ms", "s": 532, "tone": "good", "dim": false, "ci": "532–1787 p50–p90 · us-east4 direct", "note": null }, "cost": { "v": "$5.00", "s": 5, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "slug": "openai-gpt-5-6-luna", "stack": null, "pin": "openai:gpt-5.6-luna", "pinByMetric": null, "lead": false, "status": { "label": "SOLID", "tone": "good" }, "measured": "2026-07-24", "flag": "Reasoning off (0 reasoning tokens).", "cells": { "score": { "v": "88", "s": 88, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "19%", "s": 19, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "31%", "s": 31, "tone": "warn", "dim": false, "ci": null, "note": null }, "ttft": { "v": "449ms", "s": 449, "tone": "good", "dim": false, "ci": "449–598 p50–p90", "note": null }, "cost": { "v": "$1.20", "s": 1.2, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-4.1", "slug": "openai-gpt-4-1", "stack": null, "pin": "openai:gpt-4.1", "pinByMetric": null, "lead": false, "status": { "label": "STRONG", "tone": "good" }, "measured": null, "flag": "Fast enough for a live turn; priciest of the group. TTFT measured 2026-07-27.", "cells": { "score": { "v": "84", "s": 84, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "94%", "s": 94, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "9%", "s": 9, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "17%", "s": 17, "tone": "warn", "dim": false, "ci": null, "note": null }, "ttft": { "v": "640ms", "s": 640, "tone": "good", "dim": false, "ci": "640–775 p50–p90", "note": null }, "cost": { "v": "$8.00", "s": 8, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "slug": "gemini-gemini-3-5-flash-lite", "stack": null, "pin": "gemini:gemini-3.5-flash-lite", "pinByMetric": null, "lead": false, "status": { "label": "SOLID", "tone": "good" }, "measured": "2026-07-25", "flag": "Drops some compound multi-step requests.", "cells": { "score": { "v": "83", "s": 83, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "86%", "s": 86, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "3%", "s": 3, "tone": "warn", "dim": false, "ci": null, "note": null }, "deadair": { "v": "22%", "s": 22, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "52%", "s": 52, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "373ms", "s": 373, "tone": "good", "dim": false, "ci": "373–404 p50–p90", "note": null }, "cost": { "v": "$2.50", "s": 2.5, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "slug": "openai-gpt-4-1-mini", "stack": null, "pin": "openai:gpt-4.1-mini", "pinByMetric": null, "lead": false, "status": { "label": "FABRICATES", "tone": "warn" }, "measured": null, "flag": "Fabricates on 20% of out-of-policy questions (full gpt-4.1: 0%). TTFT measured 2026-07-27.", "cells": { "score": { "v": "80", "s": 80, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "20%", "s": 20, "tone": "bad", "dim": false, "ci": null, "note": null }, "deadair": { "v": "7%", "s": 7, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "14%", "s": 14, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "708ms", "s": 708, "tone": "good", "dim": false, "ci": "708–1094 p50–p90", "note": null }, "cost": { "v": "$1.60", "s": 1.6, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "Cerebras", "model": "gpt-oss-120b", "slug": "cerebras-gpt-oss-120b", "stack": null, "pin": "cerebras:gpt-oss-120b", "pinByMetric": null, "lead": false, "status": { "label": "DEAD AIR", "tone": "warn" }, "measured": null, "flag": "Silent on 100% of tool calls — dead air on every action.", "cells": { "score": { "v": "80", "s": 80, "tone": "good", "dim": false, "ci": null, "note": null }, "task": { "v": "94%", "s": 94, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "10%", "s": 10, "tone": "bad", "dim": false, "ci": null, "note": null }, "deadair": { "v": "51%", "s": 51, "tone": "bad", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "100%", "s": 100, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "195ms", "s": 195, "tone": "good", "dim": false, "ci": "195–260 p50–p90", "note": null }, "cost": { "v": "$0.75", "s": 0.75, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-5", "slug": "openai-gpt-5", "stack": null, "pin": "openai:gpt-5", "pinByMetric": null, "lead": false, "status": { "label": "SOLID", "tone": "good" }, "measured": null, "flag": null, "cells": { "score": { "v": "78", "s": 78, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "89%", "s": 89, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "4%", "s": 4, "tone": "warn", "dim": false, "ci": null, "note": null }, "deadair": { "v": "18%", "s": 18, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "32%", "s": 32, "tone": "warn", "dim": false, "ci": null, "note": null }, "ttft": { "v": "635ms", "s": 635, "tone": "good", "dim": false, "ci": "635–679 p50–p90", "note": null }, "cost": { "v": "$10.00", "s": 10, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-5-nano", "slug": "openai-gpt-5-nano", "stack": null, "pin": "openai:gpt-5-nano", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "score": { "v": "74", "s": 74, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "56%", "s": 56, "tone": "warn", "dim": false, "ci": null, "note": null }, "fabric": { "v": "3%", "s": 3, "tone": "warn", "dim": false, "ci": null, "note": null }, "deadair": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "525ms", "s": 525, "tone": "good", "dim": false, "ci": "525–1020 p50–p90", "note": null }, "cost": { "v": "$0.40", "s": 0.4, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "slug": "anthropic-claude-sonnet-5", "stack": null, "pin": "anthropic:claude-sonnet-5", "pinByMetric": null, "lead": false, "status": { "label": "SOLID", "tone": "good" }, "measured": null, "flag": "1.21s TTFT — too slow to lead a live turn.", "cells": { "score": { "v": "68", "s": 68, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "89%", "s": 89, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "7%", "s": 7, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "12%", "s": 12, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "1.21s", "s": 1206, "tone": "warn", "dim": false, "ci": "1.21–1.37s p50–p90 · us-east4 direct", "note": null }, "cost": { "v": "$10.00", "s": 10, "tone": "warn", "dim": false, "ci": null, "note": "Introductory rate through 2026-08-31; $15.00 from 2026-09-01." } } }, { "provider": "LiveKit", "model": "kimi-k2.5", "slug": "livekit-kimi-k2-5", "stack": null, "pin": "livekit:moonshotai/kimi-k2.5", "pinByMetric": null, "lead": false, "status": { "label": "SOLID", "tone": "good" }, "measured": null, "flag": "Thinking serving (not disableable) — 1.39s to first spoken token; measured 2026-07-27 on the LiveKit gateway.", "cells": { "score": { "v": "64", "s": 64, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "89%", "s": 89, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "4%", "s": 4, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "11%", "s": 11, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "1.39s", "s": 1393, "tone": "warn", "dim": false, "ci": "1.39–2.81s p50–p90", "note": null }, "cost": { "v": "$3.00", "s": 3, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-5-mini", "slug": "openai-gpt-5-mini", "stack": null, "pin": "openai:gpt-5-mini", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": null, "cells": { "score": { "v": "64", "s": 64, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "61%", "s": 61, "tone": "warn", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "25%", "s": 25, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "42%", "s": 42, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "746ms", "s": 746, "tone": "good", "dim": false, "ci": "746–794 p50–p90", "note": null }, "cost": { "v": "$2.00", "s": 2, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "Together", "model": "Llama-3.3-70B", "slug": "together-llama-3-3-70b", "stack": null, "pin": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "pinByMetric": null, "lead": false, "status": { "label": "FAIR", "tone": "warn" }, "measured": null, "flag": "Fails restraint — fires a tool with a made-up ID instead of asking for the missing one.", "cells": { "score": { "v": "62", "s": 62, "tone": "warn", "dim": false, "ci": null, "note": null }, "task": { "v": "56%", "s": 56, "tone": "warn", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "29%", "s": 29, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "42%", "s": 42, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "698ms", "s": 698, "tone": "good", "dim": false, "ci": "698–860 p50–p90", "note": null }, "cost": { "v": "$1.04", "s": 1.04, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "Alibaba", "model": "qwen-turbo", "slug": "alibaba-qwen-turbo", "stack": null, "pin": "alibaba:qwen-turbo", "pinByMetric": null, "lead": false, "status": { "label": "WEAK", "tone": "bad" }, "measured": null, "flag": "Never completes a multi-step task and fabricates on 40% of out-of-policy questions.", "cells": { "score": { "v": "49", "s": 49, "tone": "bad", "dim": false, "ci": null, "note": null }, "task": { "v": "33%", "s": 33, "tone": "bad", "dim": false, "ci": null, "note": null }, "fabric": { "v": "40%", "s": 40, "tone": "bad", "dim": false, "ci": null, "note": null }, "deadair": { "v": "29%", "s": 29, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "100%", "s": 100, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "483ms", "s": 483, "tone": "good", "dim": false, "ci": "483–497 p50–p90", "note": null }, "cost": { "v": "$0.20", "s": 0.2, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "xAI", "model": "Grok-4.3", "slug": "xai-grok-4-3", "stack": null, "pin": "xai:grok-4.3", "pinByMetric": null, "lead": false, "status": { "label": "SLOW", "tone": "warn" }, "measured": null, "flag": "2.15s TTFT — too slow for a live turn.", "cells": { "score": { "v": "43", "s": 43, "tone": "bad", "dim": false, "ci": null, "note": null }, "task": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "24%", "s": 24, "tone": "warn", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "48%", "s": 48, "tone": "bad", "dim": false, "ci": null, "note": null }, "ttft": { "v": "2.15s", "s": 2150, "tone": "warn", "dim": false, "ci": "2.15–2.64s p50–p90", "note": null }, "cost": { "v": "$2.50", "s": 2.5, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "Together", "model": "DeepSeek-V4-Pro", "slug": "together-deepseek-v4-pro", "stack": null, "pin": "together:deepseek-ai/DeepSeek-V4-Pro", "pinByMetric": null, "lead": false, "status": { "label": "NOT REAL-TIME", "tone": "warn" }, "measured": null, "flag": "1.6T reasoning model with no real-time latency measurement.", "cells": { "score": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "task": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "fabric": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "deadair": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "toolsilence": { "v": "0%", "s": 0, "tone": "good", "dim": false, "ci": null, "note": null }, "ttft": { "v": "—", "s": null, "tone": null, "dim": true, "ci": null, "note": "No real-time first-token measurement; exceeds the ~800ms voice latency budget." }, "cost": { "v": "$3.48", "s": 3.48, "tone": null, "dim": false, "ci": null, "note": null } } } ] }, "s2s": { "id": "s2s", "title": "S2S", "navLabel": "Speech-to-Speech", "subtitle": "Native voice-to-voice, ranked across six concierge scenarios.", "meta": "6 scenarios · n=3 · clean audio · latency us-east4 · 2026-07-19 (2.0 row 07-31)", "method": null, "measured": "2026-07-19", "measuredThrough": null, "laterRuns": [], "rankCol": "score", "headline": [ "score", "action", "lat" ], "scatter": { "x": "lat", "y": "score" }, "columns": [ { "key": "score", "label": "Overall", "dir": "higher", "info": "Completion × capability (action weighted), mean over 6 concierge scenarios, n=3." }, { "key": "action", "label": "Action", "dir": "higher", "info": "Fired the write tool after the caller confirmed — not just said it would. 0–1." }, { "key": "reliab", "label": "Call completion", "dir": "higher", "info": "Share of the scripted call completed before going silent or dropping." }, { "key": "cap", "label": "Task capability", "dir": "higher", "info": "Mean of 6 transcript checks: tool choice, no false calls, honest recovery, no invented prices, latest-value recall, brevity." }, { "key": "lat", "label": "Latency", "dir": "lower", "info": "End-of-speech to first audio, p50, us-east4 (reused from 2026-07-08)." } ], "rows": [ { "provider": "xAI", "model": "grok-voice-think-fast-2.0", "slug": "xai-grok-voice-think-fast-2-0", "stack": null, "pin": "xai:grok-voice-think-fast-2.0", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Fast median, but some turns stall ~3.4s.", "cells": { "score": { "v": "0.80", "s": 80, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.67", "s": 67, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "100%", "s": 100, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.86", "s": 86, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "820ms", "s": 820, "tone": "good", "dim": false, "ci": "1141·3419", "note": null }, "price": { "v": "$0.08 · min", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "Gemini", "model": "gemini-3.1-flash-live", "slug": "google-gemini-3-1-flash-live", "stack": null, "pin": "google:gemini-3.1-flash-live-preview", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "score": { "v": "0.77", "s": 77, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.78", "s": 78, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "92%", "s": 92, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.86", "s": 86, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1108ms", "s": 1108, "tone": "warn", "dim": false, "ci": "1209·1285", "note": null }, "price": { "v": "$3.00 / $12.00 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": "Google publishes both bases: $0.005/min in, $0.018/min out." } } }, { "provider": "OpenAI", "model": "gpt-realtime", "slug": "openai-gpt-realtime", "stack": null, "pin": "openai:gpt-realtime", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "score": { "v": "0.76", "s": 76, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.83", "s": 83, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "88%", "s": 88, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.87", "s": 87, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "494ms", "s": 494, "tone": "good", "dim": false, "ci": "768·870", "note": null }, "price": { "v": "$32 / $64 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1-mini", "slug": "openai-gpt-realtime-2-1-mini", "stack": null, "pin": "openai:gpt-realtime-2.1-mini", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "score": { "v": "0.75", "s": 75, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.83", "s": 83, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "93%", "s": 93, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.79", "s": 79, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1011ms", "s": 1011, "tone": "warn", "dim": false, "ci": "1360·1542", "note": null }, "price": { "v": "$10 / $20 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "xAI", "model": "grok-voice-fast", "slug": "xai-grok-voice-fast", "stack": null, "pin": "xai:grok-voice-fast-1.0", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Fires the booking tool <50% of the time.", "cells": { "score": { "v": "0.72", "s": 72, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.44", "s": 44, "tone": "warn", "dim": false, "ci": null, "note": null }, "reliab": { "v": "95%", "s": 95, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.91", "s": 91, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1111ms", "s": 1111, "tone": "warn", "dim": false, "ci": "1576·2752", "note": null }, "price": { "v": "$0.05 · min", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-2", "slug": "openai-gpt-realtime-2", "stack": null, "pin": "openai:gpt-realtime-2", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Strong per turn, inconsistent across scenarios.", "cells": { "score": { "v": "0.68", "s": 68, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.87", "s": 87, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "81%", "s": 81, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.84", "s": 84, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1103ms", "s": 1103, "tone": "warn", "dim": false, "ci": "1468·1626", "note": null }, "price": { "v": "$32 / $64 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1", "slug": "openai-gpt-realtime-2-1", "stack": null, "pin": "openai:gpt-realtime-2.1", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "score": { "v": "0.63", "s": 63, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.56", "s": 56, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "90%", "s": 90, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.76", "s": 76, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1104ms", "s": 1104, "tone": "warn", "dim": false, "ci": "1514·2146", "note": null }, "price": { "v": "$32 / $64 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-mini", "slug": "openai-gpt-realtime-mini", "stack": null, "pin": "openai:gpt-realtime-mini", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Abandons ~26% of calls — goes silent at the booking chain (turns 6–8).", "cells": { "score": { "v": "0.62", "s": 62, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.70", "s": 70, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "74%", "s": 74, "tone": "warn", "dim": false, "ci": null, "note": null }, "cap": { "v": "0.91", "s": 91, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "614ms", "s": 614, "tone": "good", "dim": false, "ci": "812·847", "note": null }, "price": { "v": "$10 / $20 · 1M tok", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "xAI", "model": "grok-voice-think-fast-1.0", "slug": "xai-grok-voice-think-fast", "stack": null, "pin": "xai:grok-voice-think-fast-1.0", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Finishes every call, rarely commits an action (0.22) — mostly narrates.", "cells": { "score": { "v": "0.56", "s": 56, "tone": null, "dim": false, "ci": null, "note": null }, "action": { "v": "0.22", "s": 22, "tone": "bad", "dim": false, "ci": null, "note": null }, "reliab": { "v": "96%", "s": 96, "tone": null, "dim": false, "ci": null, "note": null }, "cap": { "v": "0.77", "s": 77, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "1007ms", "s": 1007, "tone": "warn", "dim": false, "ci": "1360·1426", "note": null }, "price": { "v": "$0.05 · min", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } }, { "provider": "Inworld", "model": "gpt-4.1-nano (cascade)", "slug": "inworld-gpt-4-1-nano-cascade", "stack": null, "pin": "inworld:openai/gpt-4.1-nano", "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": "Cascade ASR+LLM+TTS — fires tools fine, drops ~49% of calls early.", "cells": { "score": { "v": "0.45", "s": 45, "tone": "bad", "dim": false, "ci": null, "note": null }, "action": { "v": "1.00", "s": 100, "tone": null, "dim": false, "ci": null, "note": null }, "reliab": { "v": "51%", "s": 51, "tone": "bad", "dim": false, "ci": null, "note": null }, "cap": { "v": "0.86", "s": 86, "tone": null, "dim": false, "ci": null, "note": null }, "lat": { "v": "—", "s": null, "tone": null, "dim": false, "ci": null, "note": null }, "price": { "v": "—", "s": null, "tone": null, "dim": false, "ci": null, "note": null } } } ] }, "stacks": { "id": "stacks", "title": "Cost per solve / whole stack", "navLabel": "Cost per solve / whole stack", "subtitle": "Cost per grounded solve across a full STT+LLM+TTS stack.", "meta": "dental-booking task · n=3/stack · grounded (tool fired) · measured 2026-07-03", "method": null, "measured": "2026-07-03", "measuredThrough": null, "laterRuns": [], "rankCol": "solve", "headline": [ "solve", "grounded", "call" ], "scatter": null, "columns": [ { "key": "grounded", "label": "Grounded", "dir": "higher", "info": "% of runs the required tool actually fired. Higher is better." }, { "key": "turns", "label": "Turns", "dir": null, "info": "Mean turns to resolution." }, { "key": "call", "label": "$/call", "dir": "lower", "info": "Cost per call: real consumption × billed rates." }, { "key": "solve", "label": "$/solve", "dir": "lower", "info": "Cost per grounded solve. Lower = cheaper." } ], "rows": [ { "provider": "", "model": "CHEAPEST", "slug": "cheapest", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": true, "status": { "label": "GROUNDED", "tone": "good" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "turns": { "v": "5.3", "s": 5.3, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.058", "s": 0.058, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.058", "s": 0.058, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "DEEPGRAM-NATIVE", "slug": "deepgram-native", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "deepgram:aura-2" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "GROUNDED", "tone": "good" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good", "dim": false, "ci": null, "note": null }, "turns": { "v": "5.0", "s": 5, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.123", "s": 0.123, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.123", "s": 0.123, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "SINGLE-VENDOR", "slug": "single-vendor", "stack": [ "cartesia:ink-2", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": "Cheap only because grounding is weak (67%).", "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "6.0", "s": 6, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.087", "s": 0.087, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.131", "s": 0.131, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "EXPRESSIVE VOICE", "slug": "expressive-voice", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "minimax:speech-2.6-hd" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "4.3", "s": 4.3, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.092", "s": 0.092, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.137", "s": 0.137, "tone": "good", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "PREMIUM VOICE / CHEAP BRAIN", "slug": "premium-voice-cheap-brain", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "elevenlabs:eleven_flash_v2_5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "WEAK", "tone": "bad" }, "measured": null, "flag": "Only 33% grounded — rarely closes the booking despite low cost.", "cells": { "grounded": { "v": "33%", "s": 33, "tone": "bad", "dim": false, "ci": null, "note": null }, "turns": { "v": "1.7", "s": 1.7, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.055", "s": 0.055, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "PREMIUM BRAIN / CHEAP VOICE", "slug": "premium-brain-cheap-voice", "stack": [ "deepgram:nova-3", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "4.3", "s": 4.3, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.111", "s": 0.111, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "CHEAP-OPENAI", "slug": "cheap-openai", "stack": [ "deepgram:nova-3", "openai:gpt-5-nano", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "10.0", "s": 10, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.145", "s": 0.145, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.218", "s": 0.218, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "ACCURACY", "slug": "accuracy", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "5.7", "s": 5.7, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.147", "s": 0.147, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.221", "s": 0.221, "tone": "warn", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "PREMIUM (ALL-BRAND)", "slug": "premium-all-brand", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "elevenlabs:eleven_flash_v2_5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "PARTIAL", "tone": "warn" }, "measured": null, "flag": null, "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn", "dim": false, "ci": null, "note": null }, "turns": { "v": "7.3", "s": 7.3, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.469", "s": 0.469, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "$0.703", "s": 0.703, "tone": "bad", "dim": false, "ci": null, "note": null } } }, { "provider": "", "model": "BALANCED", "slug": "balanced", "stack": [ "deepgram:nova-3", "openai:gpt-4.1-mini", "cartesia:sonic-3.5" ], "pin": null, "pinByMetric": null, "lead": false, "status": { "label": "NO SOLVE", "tone": "bad" }, "measured": null, "flag": "Never booked (0% grounded) — gpt-4.1-mini kept clarifying and never fired the tool.", "cells": { "grounded": { "v": "0%", "s": 0, "tone": "bad", "dim": false, "ci": null, "note": null }, "turns": { "v": "3.3", "s": 3.3, "tone": null, "dim": true, "ci": null, "note": null }, "call": { "v": "$0.079", "s": 0.079, "tone": null, "dim": false, "ci": null, "note": null }, "solve": { "v": "never booked", "s": null, "tone": "bad", "dim": false, "ci": null, "note": null } } } ] }, "turntaking": { "id": "turntaking", "title": "Turn-taking · end-of-turn detection", "navLabel": "Turn-taking", "subtitle": "End-of-turn detection, ranked on 200 real human clips.", "meta": "English · 200 real human clips · pipecat smart-turn-v3.1-test · transcripts via Deepgram nova-3", "method": null, "measured": null, "measuredThrough": null, "laterRuns": [], "rankCol": "acc", "headline": [ "acc", "cut", "lat" ], "scatter": null, "columns": [ { "key": "acc", "label": "EOT accuracy", "dir": "higher", "info": "Correct END-vs-WAIT decisions across 200 real clips, at each model’s best operating threshold (min false-cutoff s.t. END-recall ≥ 85%)." }, { "key": "recall", "label": "END-recall", "dir": "higher", "info": "Completed turns correctly ended — the agent didn’t leave the caller hanging." }, { "key": "cut", "label": "False-cutoff", "dir": "lower", "info": "WAIT clips wrongly ended = talked over the caller mid-thought. The costly live-call error." }, { "key": "lat", "label": "Inference", "dir": "lower", "info": "Model forward pass, warm CPU. Text models additionally wait on the STT transcript; audio runs in parallel with STT." } ], "rows": [ { "provider": "Smart Turn v3.2", "model": "Pipecat · Whisper-tiny", "slug": "smart-turn-v3-2-pipecat-whisper-tiny", "stack": null, "pin": null, "pinByMetric": null, "lead": true, "status": null, "measured": null, "flag": "Eval set is Smart Turn’s home distribution — treat 94% as a ceiling.", "cells": { "acc": { "v": "94.0%", "s": 94, "tone": "good", "dim": false, "ci": null, "note": null }, "recall": { "v": "89.0%", "s": 89, "tone": "good", "dim": false, "ci": null, "note": null }, "cut": { "v": "1.0%", "s": 1, "tone": "good", "dim": false, "ci": null, "note": "One false cutoff in 100 incomplete clips." }, "lat": { "v": "32ms", "s": 32, "tone": "good", "dim": false, "ci": "49 p90", "note": "Runs in parallel with STT — no transcript wait." } } }, { "provider": "LiveKit Intl", "model": "turn-detector v0.4.1", "slug": "livekit-intl-turn-detector-v0-4-1", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "acc": { "v": "87.0%", "s": 87, "tone": "good", "dim": false, "ci": null, "note": null }, "recall": { "v": "86.0%", "s": 86, "tone": "good", "dim": false, "ci": null, "note": null }, "cut": { "v": "12.0%", "s": 12, "tone": "warn", "dim": false, "ci": null, "note": null }, "lat": { "v": "29ms", "s": 29, "tone": "good", "dim": false, "ci": "57 p90", "note": "Waits for the STT transcript (~0–125ms) before running." } } }, { "provider": "Turnsense", "model": "SmolLM2-135M", "slug": "turnsense-smollm2-135m", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "acc": { "v": "86.0%", "s": 86, "tone": "good", "dim": false, "ci": null, "note": null }, "recall": { "v": "88.0%", "s": 88, "tone": "good", "dim": false, "ci": null, "note": null }, "cut": { "v": "16.0%", "s": 16, "tone": "warn", "dim": false, "ci": null, "note": null }, "lat": { "v": "127ms", "s": 127, "tone": "warn", "dim": false, "ci": "193 p90", "note": "Pads every input to 256 tokens regardless of length — slowest as-shipped." } } }, { "provider": "LiveKit EN", "model": "turn-detector v1.2.2", "slug": "livekit-en-turn-detector-v1-2-2", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "acc": { "v": "82.0%", "s": 82, "tone": "warn", "dim": false, "ci": null, "note": null }, "recall": { "v": "87.0%", "s": 87, "tone": "good", "dim": false, "ci": null, "note": null }, "cut": { "v": "23.0%", "s": 23, "tone": "warn", "dim": false, "ci": null, "note": null }, "lat": { "v": "5ms", "s": 5, "tone": "good", "dim": false, "ci": "24 p90", "note": null } } }, { "provider": "VAD + silence timer", "model": "baseline", "slug": "vad-silence-timer-baseline", "stack": null, "pin": null, "pinByMetric": null, "lead": false, "status": null, "measured": null, "flag": null, "cells": { "acc": { "v": "46.9%", "s": 46.9, "tone": "bad", "dim": true, "ci": null, "note": null }, "recall": { "v": "100%", "s": 100, "tone": "neutral", "dim": true, "ci": null, "note": null }, "cut": { "v": "100%", "s": 100, "tone": "bad", "dim": true, "ci": null, "note": "A silence timer ends every turn — the floor semantic detection has to beat." }, "lat": { "v": "~1ms", "s": 1, "tone": "neutral", "dim": true, "ci": null, "note": null } } } ] } } }