little-tts / evaluation /gpu_batch_benchmark.json
TuringLogic's picture
Add PyTorch-free FP32 GPU batch graphs, queue, validation and RTX 5090 benchmarks
550ce03 verified
Raw History Blame Contribute Delete
15.2 kB
{
"measured_at": "2026-11-16T18:00:16.016026+00:00",
"device": "cuda",
"gpu": "NVIDIA GeForce RTX 5090, 580.178.04, 32607",
"platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39",
"python": "3.12.3",
"onnxruntime": "1.32.0",
"precision": "FP32",
"tf32": false,
"quantized": false,
"pytorch_installed": false,
"pytorch_imported": false,
"results": [
{
"target_clip_seconds": 5.0,
"batch_size": 1,
"actual_mean_clip_seconds": 5.002666666666666,
"median_batch_ms": 23.545040006865747,
"p95_batch_ms": 23.599622648907825,
"min_batch_ms": 23.443258018232882,
"max_batch_ms": 23.719657998299226,
"clips_per_second": 42.47179022453984,
"audio_seconds_per_second": 212.47220922996462,
"original_serial_median_ms": 54.86346798716113,
"speedup_over_original_serial": 2.330149703341464,
"gpu_resident_mib_after_run": 1304.0,
"repeats": 20,
"timings_ms": [
23.4944429830648,
23.719657998299226,
23.51024598465301,
23.547132004750893,
23.58465999714099,
23.589938995428383,
23.444332997314632,
23.54565600398928,
23.544424009742215,
23.471412976505235,
23.50356601527892,
23.443258018232882,
23.593304998939857,
23.565187002532184,
23.542345996247604,
23.566295014461502,
23.53850399958901,
23.55438601807691,
23.49075098754838,
23.55722399079241
]
},
{
"target_clip_seconds": 5.0,
"batch_size": 2,
"actual_mean_clip_seconds": 5.008,
"median_batch_ms": 24.75920398137532,
"p95_batch_ms": 24.86369372054469,
"min_batch_ms": 24.65680698514916,
"max_batch_ms": 24.86592999775894,
"clips_per_second": 80.77804122880788,
"audio_seconds_per_second": 404.53643047386987,
"original_serial_median_ms": 110.53136200644076,
"speedup_over_original_serial": 4.46425345861628,
"gpu_resident_mib_after_run": 1364.0,
"repeats": 20,
"timings_ms": [
24.746326002059504,
24.86592999775894,
24.65680698514916,
24.765748996287584,
24.69861801364459,
24.78549600346014,
24.754590005613863,
24.72542499890551,
24.736111983656883,
24.796628014883026,
24.84475402161479,
24.86357602174394,
24.75805499125272,
24.760352971497923,
24.754782003583387,
24.781468004221097,
24.739813001360744,
24.7890230093617,
24.787250004010275,
24.754844984272495
]
},
{
"target_clip_seconds": 5.0,
"batch_size": 4,
"actual_mean_clip_seconds": 5.002666666666666,
"median_batch_ms": 27.277934001176618,
"p95_batch_ms": 27.329145339899696,
"min_batch_ms": 27.16956500080414,
"max_batch_ms": 27.625590009847656,
"clips_per_second": 146.63867138279102,
"audio_seconds_per_second": 733.5843933709758,
"original_serial_median_ms": 220.83054602262564,
"speedup_over_original_serial": 8.095574467373527,
"gpu_resident_mib_after_run": 1430.0,
"repeats": 20,
"timings_ms": [
27.3003299953416,
27.625590009847656,
27.274645981378853,
27.303933980874717,
27.288220007903874,
27.220826013945043,
27.290136000374332,
27.229689003434032,
27.298950997646898,
27.16956500080414,
27.22632500808686,
27.313542988849804,
27.208749001147225,
27.231224987190217,
27.291100996080786,
27.284671989036724,
27.270089980447665,
27.281222020974383,
27.251658000750467,
27.22551499027759
]
},
{
"target_clip_seconds": 5.0,
"batch_size": 8,
"actual_mean_clip_seconds": 5.002666666666666,
"median_batch_ms": 32.72439949796535,
"p95_batch_ms": 33.01015651086345,
"min_batch_ms": 32.634268020046875,
"max_batch_ms": 33.338790002744645,
"clips_per_second": 244.465906868586,
"audio_seconds_per_second": 1222.9814434279128,
"original_serial_median_ms": 441.6329039959237,
"speedup_over_original_serial": 13.495523547296333,
"gpu_resident_mib_after_run": 1726.0,
"repeats": 20,
"timings_ms": [
32.65888901660219,
33.338790002744645,
32.717281981604174,
32.746787008363754,
32.7643140044529,
32.68697200110182,
32.68676099833101,
32.71354699973017,
32.75515299174003,
32.716526999138296,
32.772487989859655,
32.634268020046875,
32.771514990599826,
32.70338400034234,
32.73151701432653,
32.713235006667674,
32.80776899191551,
32.99286001129076,
32.78051401139237,
32.66763599822298
]
},
{
"target_clip_seconds": 5.0,
"batch_size": 16,
"actual_mean_clip_seconds": 5.002666666666666,
"median_batch_ms": 44.24853149976116,
"p95_batch_ms": 44.44759108882863,
"min_batch_ms": 44.03272899799049,
"max_batch_ms": 45.71626099641435,
"clips_per_second": 361.5939209211127,
"audio_seconds_per_second": 1808.933855061353,
"original_serial_median_ms": 884.023568010889,
"speedup_over_original_serial": 19.97859675898308,
"gpu_resident_mib_after_run": 2204.0,
"repeats": 20,
"timings_ms": [
44.16715700062923,
45.71626099641435,
44.19082900858484,
44.052145996829495,
44.09176699118689,
44.186583982082084,
44.03272899799049,
44.26428198348731,
44.38081898842938,
44.29361698566936,
44.21786800958216,
44.2058690241538,
44.15679498924874,
44.25228800391778,
44.24477499560453,
44.26230100216344,
44.272606988670304,
44.3543290020898,
44.28456799359992,
44.288187025813386
]
},
{
"target_clip_seconds": 5.0,
"batch_size": 32,
"actual_mean_clip_seconds": 5.002666666666666,
"median_batch_ms": 69.65748649963643,
"p95_batch_ms": 69.992385696969,
"min_batch_ms": 69.4775779847987,
"max_batch_ms": 72.13447798858397,
"clips_per_second": 459.39067870568397,
"audio_seconds_per_second": 2298.1784353383014,
"original_serial_median_ms": 1764.2975799972191,
"speedup_over_original_serial": 25.328183209803697,
"gpu_resident_mib_after_run": 3116.0,
"repeats": 20,
"timings_ms": [
69.4775779847987,
72.13447798858397,
69.76285998825915,
69.69329100684263,
69.74769700900652,
69.8567460058257,
69.87964399741031,
69.60785499541089,
69.62381300400011,
69.6312109939754,
69.5107109786477,
69.65030598803423,
69.63481401908211,
69.64816199615598,
69.64666000567377,
69.66466701123863,
69.67188100679778,
69.63217101292685,
69.78108701878227,
69.76907601347193
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 1,
"actual_mean_clip_seconds": 10.048,
"median_batch_ms": 26.200769512797706,
"p95_batch_ms": 26.291609500185587,
"min_batch_ms": 26.027974003227428,
"max_batch_ms": 26.29967499524355,
"clips_per_second": 38.16681794447115,
"audio_seconds_per_second": 383.50018670604607,
"original_serial_median_ms": 79.45163999102078,
"speedup_over_original_serial": 3.0324162789269535,
"gpu_resident_mib_after_run": 3372.0,
"repeats": 20,
"timings_ms": [
26.126906974241138,
26.15752097335644,
26.291185000445694,
26.10891798394732,
26.23674599453807,
26.234860997647047,
26.149765006266534,
26.264323998475447,
26.21950799948536,
26.192582008661702,
26.139434979995713,
26.226441987091675,
26.164954993873835,
26.27940400270745,
26.125644013518468,
26.2536030204501,
26.027974003227428,
26.29967499524355,
26.20895701693371,
26.176981016760692
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 2,
"actual_mean_clip_seconds": 10.021333333333333,
"median_batch_ms": 28.707453006063588,
"p95_batch_ms": 28.814201983914245,
"min_batch_ms": 28.633770998567343,
"max_batch_ms": 28.83656500489451,
"clips_per_second": 69.66831921931771,
"audio_seconds_per_second": 698.1694496698559,
"original_serial_median_ms": 158.78587300539948,
"speedup_over_original_serial": 5.5311724440291075,
"gpu_resident_mib_after_run": 3372.0,
"repeats": 20,
"timings_ms": [
28.684730001259595,
28.83656500489451,
28.633770998567343,
28.75325400964357,
28.691974992398173,
28.671734005911276,
28.723941009957343,
28.736412001308054,
28.7899780087173,
28.81302498281002,
28.667895996477455,
28.72641902649775,
28.75903999665752,
28.695115994196385,
28.71809300268069,
28.667523991316557,
28.708978003123775,
28.663848992437124,
28.7059280090034,
28.641621000133455
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 4,
"actual_mean_clip_seconds": 10.008,
"median_batch_ms": 33.918748013093136,
"p95_batch_ms": 34.141711839765776,
"min_batch_ms": 33.830186002887785,
"max_batch_ms": 34.16116500739008,
"clips_per_second": 117.92888105586742,
"audio_seconds_per_second": 1180.232241607121,
"original_serial_median_ms": 317.45494800270535,
"speedup_over_original_serial": 9.359276700901905,
"gpu_resident_mib_after_run": 3372.0,
"repeats": 20,
"timings_ms": [
33.875622000778094,
33.83843699702993,
33.92082900973037,
33.98885700153187,
33.87033601757139,
33.916667016455904,
33.87041701353155,
33.92905497457832,
33.96815399173647,
33.890516991959885,
33.87488398584537,
33.830186002887785,
33.89702399726957,
33.92980701755732,
33.90145598677918,
33.95272998022847,
34.06847800943069,
34.16116500739008,
34.14068798883818,
33.99262498714961
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 8,
"actual_mean_clip_seconds": 10.008,
"median_batch_ms": 47.943347512045875,
"p95_batch_ms": 48.18261180917034,
"min_batch_ms": 47.75105998851359,
"max_batch_ms": 48.29930601408705,
"clips_per_second": 166.86360913763858,
"audio_seconds_per_second": 1669.9710002494867,
"original_serial_median_ms": 635.7034779794049,
"speedup_over_original_serial": 13.259472084624106,
"gpu_resident_mib_after_run": 3372.0,
"repeats": 20,
"timings_ms": [
47.82059500575997,
48.00595997949131,
47.97201798646711,
47.93980001704767,
47.97892100759782,
47.75105998851359,
47.929656982887536,
47.85619699396193,
47.90211102226749,
47.94689500704408,
47.92148602427915,
47.874184005195275,
47.98592900624499,
47.89770900970325,
47.899830009555444,
48.176470008911565,
48.29930601408705,
47.978513000998646,
48.0153450043872,
48.0713669967372
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 16,
"actual_mean_clip_seconds": 10.008,
"median_batch_ms": 73.34223299403675,
"p95_batch_ms": 74.52959322254173,
"min_batch_ms": 73.1955660157837,
"max_batch_ms": 76.06502599082887,
"clips_per_second": 218.15534306544654,
"audio_seconds_per_second": 2183.2986733989887,
"original_serial_median_ms": 1269.3672360037453,
"speedup_over_original_serial": 17.30745280290217,
"gpu_resident_mib_after_run": 4054.0,
"repeats": 20,
"timings_ms": [
73.38895200518891,
76.06502599082887,
74.44878097157925,
73.20771101512946,
73.30435101175681,
73.22891900548711,
73.33209499483928,
73.2174550066702,
73.51771800313145,
73.8643050135579,
73.30944100976922,
73.2786699954886,
73.42671501100995,
73.31994498963468,
73.1955660157837,
73.24376501492225,
73.35237099323422,
74.27624199772254,
73.9177439827472,
73.42872497974895
]
},
{
"target_clip_seconds": 10.0,
"batch_size": 32,
"actual_mean_clip_seconds": 10.008,
"median_batch_ms": 157.35362051054835,
"p95_batch_ms": 161.22798981668893,
"min_batch_ms": 156.00377900409512,
"max_batch_ms": 163.1119690136984,
"clips_per_second": 203.36360800706743,
"audio_seconds_per_second": 2035.2629889347306,
"original_serial_median_ms": 2539.717275009025,
"speedup_over_original_serial": 16.140189636366024,
"gpu_resident_mib_after_run": 4106.0,
"repeats": 20,
"timings_ms": [
157.0540610118769,
161.12883301684633,
157.00360998744145,
157.6531800092198,
163.1119690136984,
156.36324402294122,
159.70016800565645,
156.25232399906963,
159.4500499777496,
156.32348600775003,
159.83640399645083,
156.1454539769329,
160.53772901068442,
156.2099059810862,
159.7276270040311,
156.53526401729323,
159.53245598939247,
156.00377900409512,
159.92580002057366,
156.14569798344746
]
}
],
"simultaneous_bursts": [
{
"target_clip_seconds": 5.0,
"simultaneous_requests": 64,
"max_batch": 32,
"queue_wait_ms": 2,
"p50_completion_ms": 119.94645500089973,
"p95_completion_ms": 157.02242639963515,
"all_completed_ms": 157.0239359862171,
"finished_within_90ms": 32,
"finished_within_500ms": 64
},
{
"target_clip_seconds": 10.0,
"simultaneous_requests": 64,
"max_batch": 32,
"queue_wait_ms": 2,
"p50_completion_ms": 260.04735700553283,
"p95_completion_ms": 342.8289098534151,
"all_completed_ms": 342.83116098959,
"finished_within_90ms": 0,
"finished_within_500ms": 64
}
],
"scope": "Measure FP32 batch latency, serial CUDA comparison, and a simultaneous burst.\n\nTimings include phonemization, host/device transfers and audio returning to CPU.\nThey exclude loading/warm-up, WAV encoding, HTTP/network time and queueing unless\nexplicitly marked as the burst test.\n",
"memory_note": "Per-process resident VRAM sampled after each run; not a sampled peak. Includes both comparison sessions."
}