From 1eb7e9afd83fce8ca031fe193e437419a61bca4e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 22 Apr 2024 21:52:33 +0000 Subject: [PATCH] Update latest data --- website/public/data/latest.json | 1390 +++++++++++++++++-------------- 1 file changed, 776 insertions(+), 614 deletions(-) diff --git a/website/public/data/latest.json b/website/public/data/latest.json index da2024b..0890c40 100644 --- a/website/public/data/latest.json +++ b/website/public/data/latest.json @@ -1,1317 +1,1479 @@ [ { - "time": "2024-04-22T08:06:56.775212", + "time": "2024-04-22T21:48:38.795478", "region": "cdg", "cmd": "--max-tokens=20", "results": [ { "model": "openai/gpt-4-turbo", - "ttr": 0.5983378887176514, - "ttft": 0.5985531806945801, - "tps": 22.638461402979495, + "ttr": 0.7535204887390137, + "ttft": 0.7535901069641113, + "tps": 33.49172139576969, "num_tokens": 20, - "total_time": 1.4378328323364258, - "output": "Sunset paints the sky with grace,\nHorizon blushes in embrace,\nColors blend in soft ballet" + "total_time": 1.320894479751587, + "output": "Amber hues paint the evening sky,\nHorizon blushes as sun descends,\nClouds like" }, { "model": "openai/gpt-4-0125-preview", - "ttr": 0.4481072425842285, - "ttft": 0.5165748596191406, - "tps": 25.597500244116784, + "ttr": 0.37622570991516113, + "ttft": 0.4755513668060303, + "tps": 38.38135157719325, "num_tokens": 20, - "total_time": 1.2588348388671875, - "output": "Beneath the vast, expansive sky at dusk,\nThe sun dips low, in shades of gold" + "total_time": 0.970583438873291, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { "model": "azure.scentralus/gpt-4-0125-preview", - "ttr": 0.7193028926849365, - "ttft": 0.7193951606750488, - "tps": 6.5974639115838665, + "ttr": 0.646981954574585, + "ttft": 0.6472129821777344, + "tps": 12.14537425329483, "num_tokens": 20, - "total_time": 3.599289655685425, + "total_time": 2.211594581604004, "output": "Amidst the tranquil evening sky,\nA canvas painted with goodbye,\nThe sun dips low, in" }, { "model": "openai/gpt-4-1106-preview", - "ttr": 0.5919198989868164, - "ttft": 0.6541166305541992, - "tps": 19.11289186071332, + "ttr": 0.42490696907043457, + "ttft": 0.4987783432006836, + "tps": 27.218149855988493, "num_tokens": 20, - "total_time": 1.648210048675537, + "total_time": 1.1968419551849365, "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "azure.westus/gpt-4-1106-preview", - "ttr": 0.7622711658477783, - "ttft": 0.7623574733734131, - "tps": 17.539960571734067, + "ttr": 0.8249218463897705, + "ttft": 0.825002908706665, + "tps": 21.94627537128696, "num_tokens": 20, - "total_time": 1.8455982208251953, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with hues of gold and pink" + "total_time": 1.69075345993042, + "output": "Amber hues dance across the evening sky,\nHorizon ablaze with the day's last kiss,\n" }, { "model": "azure.eastus2/gpt-4-1106-preview", - "ttr": 0.9814193248748779, - "ttft": 0.981525182723999, - "tps": 10.94384937511999, + "ttr": 0.32964563369750977, + "ttft": 0.33638501167297363, + "tps": 21.993833915342325, "num_tokens": 20, - "total_time": 2.7176601886749268, - "output": "Amber hues dance across the evening sky,\nHorizon ablaze with the day's last sigh,\n" + "total_time": 1.200263500213623, + "output": "Amber hues dance across the evening sky,\nHorizon ablaze with the day's last kiss,\n" }, { "model": "azure.frcentral/gpt-4-1106-preview", - "ttr": 0.3211376667022705, - "ttft": 0.3212122917175293, - "tps": 6.335727059558323, + "ttr": 0.30475687980651855, + "ttft": 0.30536365509033203, + "tps": 11.628252653579908, "num_tokens": 20, - "total_time": 3.3200788497924805, + "total_time": 1.9393150806427002, "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { "model": "azure.secentral/gpt-4-1106-preview", - "ttr": 0.45921754837036133, - "ttft": 0.459369421005249, - "tps": 10.738073646802459, + "ttr": 0.36137843132019043, + "ttft": 0.3615248203277588, + "tps": 13.850969737461407, "num_tokens": 20, - "total_time": 2.228774309158325, + "total_time": 1.7332699298858643, "output": "Golden orb descends in the sky,\nPainting hues of red and gold,\nSilhouettes of" }, { "model": "azure.uksouth/gpt-4-1106-preview", - "ttr": 0.660820722579956, - "ttft": 0.6610517501831055, - "tps": 9.109648158683815, + "ttr": 0.3656773567199707, + "ttft": 0.36576271057128906, + "tps": 18.443400573447605, "num_tokens": 20, - "total_time": 2.7467525005340576, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 1.3959414958953857, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { "model": "openai/gpt-3.5-turbo-0125", - "ttr": 0.3197042942047119, - "ttft": 0.33257627487182617, - "tps": 67.47267667261876, + "ttr": 0.2888963222503662, + "ttft": 0.30743980407714844, + "tps": 59.39917607747585, "num_tokens": 20, - "total_time": 0.6141717433929443, - "output": "The sun dips low, painting the sky in hues\nOf orange, pink, and purple, so" + "total_time": 0.6273095607757568, + "output": "The sun dips below the horizon line\nPainting the sky in hues divine\nOrange, pink," }, { "model": "openai/gpt-3.5-turbo-1106", - "ttr": 0.28953075408935547, - "ttft": 0.3369176387786865, - "tps": 81.17566719634927, + "ttr": 0.2779264450073242, + "ttft": 0.304959774017334, + "tps": 86.85430294681179, "num_tokens": 20, - "total_time": 0.5709779262542725, + "total_time": 0.523716926574707, "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { "model": "azure.westus/gpt-3.5-turbo-1106", - "ttr": 0.18707895278930664, - "ttft": 0.18723368644714355, - "tps": 129.634295957823, + "ttr": 0.21573829650878906, + "ttft": 0.22484922409057617, + "tps": 132.95635011336657, "num_tokens": 20, - "total_time": 0.3337998390197754, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" + "total_time": 0.367753267288208, + "output": "The sun dips low, casting golden light\nAcross the sky, a fiery display\nColors blend," }, { "model": "azure.eastus2/gpt-3.5-turbo", - "ttr": 0.15187859535217285, - "ttft": 0.1524338722229004, - "tps": 47.618031708397, + "ttr": 0.16796040534973145, + "ttft": 0.1680440902709961, + "tps": 58.719817146629744, "num_tokens": 20, - "total_time": 0.5514423847198486, + "total_time": 0.49161458015441895, "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" }, { "model": "anthropic/claude-3-opus-20240229", - "ttr": 0.9283649921417236, - "ttft": 0.9285275936126709, - "tps": 37.925149824560236, + "ttr": 1.1693737506866455, + "ttft": 1.1696534156799316, + "tps": 37.716044129959144, "num_tokens": 20, - "total_time": 1.4295144081115723, + "total_time": 1.6734178066253662, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { "model": "anthropic/claude-3-sonnet-20240229", - "ttr": 0.5854909420013428, - "ttft": 0.5857160091400146, - "tps": 73.09636415835573, + "ttr": 0.6586403846740723, + "ttft": 0.6598470211029053, + "tps": 82.00092196246295, "num_tokens": 20, - "total_time": 0.845646858215332, + "total_time": 0.8915517330169678, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { "model": "anthropic/claude-3-haiku-20240307", - "ttr": 0.35446763038635254, - "ttft": 0.3547849655151367, - "tps": 94.45667701016728, + "ttr": 0.3497891426086426, + "ttft": 0.3548464775085449, + "tps": 94.08410357639131, "num_tokens": 20, - "total_time": 0.5559353828430176, + "total_time": 0.5567934513092041, "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," }, { "model": "anthropic/claude-2.1", - "ttr": 0.44562673568725586, - "ttft": 0.4463803768157959, - "tps": 31.241005915234442, + "ttr": 0.4326310157775879, + "ttft": 0.4391610622406006, + "tps": 29.514641810082384, "num_tokens": 20, - "total_time": 1.0545554161071777, - "output": "Here is a nonet poem about a sunset:\n\nSun descending\nGolden horizon glowing\nCl" + "total_time": 1.082909345626831, + "output": "Here is a nonet poem about a sunset:\n\nRadiant, glowing\nColors fill the" }, { "model": "anthropic/claude-instant-1.2", - "ttr": 0.5597796440124512, - "ttft": 0.5632693767547607, - "tps": 152.40289002528198, + "ttr": 0.483731746673584, + "ttft": 0.4969778060913086, + "tps": 196.61980365597606, "num_tokens": 20, - "total_time": 0.687938928604126, + "total_time": 0.5936110019683838, "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" }, { "model": "cohere/command-r-plus", - "ttr": 0.11967277526855469, - "ttft": 0.22998404502868652, - "tps": 29.418396650515717, + "ttr": 0.1194467544555664, + "ttft": 0.30515217781066895, + "tps": 25.63250215040128, "num_tokens": 17, - "total_time": 0.7738614082336426, + "total_time": 0.9293596744537354, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { "model": "cohere/command-r", - "ttr": 0.13445353507995605, - "ttft": 0.18753862380981445, - "tps": 92.68372855402909, - "num_tokens": 18, - "total_time": 0.3709580898284912, - "output": "Orange and pink skies, so serene\nThe sun melts into the sea\nSoftly the stars begin" + "ttr": 0.1367201805114746, + "ttft": 0.21966981887817383, + "tps": 80.15190302781298, + "num_tokens": 17, + "total_time": 0.41929078102111816, + "output": "Orange and pink skies,\nThe sun melts into the sea,\nPeaceful birds sing." }, { "model": "cohere/command-light", - "ttr": 0.13283348083496094, - "ttft": 0.18692946434020996, - "tps": 80.46472010129303, + "ttr": 0.13176202774047852, + "ttft": 0.23911762237548828, + "tps": 81.46605884033133, "num_tokens": 18, - "total_time": 0.39820218086242676, + "total_time": 0.44779348373413086, "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { "model": "azure.eastus2/mistral", - "ttr": 0.9695878028869629, - "ttft": 0.9696927070617676, + "ttr": 0.892432689666748, + "ttft": 0.8925323486328125, "tps": 999, "num_tokens": 20, - "total_time": 0.9700102806091309, - "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" + "total_time": 0.8927502632141113, + "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink," + }, + { + "model": "anyscale.com/Mixtral-8x7B-Instruct-v0.1", + "ttr": 0.5504801273345947, + "ttft": 0.5688138008117676, + "tps": 18.206810963870257, + "num_tokens": 20, + "total_time": 1.6123793125152588, + "output": " Sunset's glow, soon to wane,\nDay's fiery orb in night" }, { "model": "fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.20833659172058105, - "ttft": 0.20839643478393555, - "tps": 170.47280817156, + "ttr": 0.28383541107177734, + "ttft": 0.2839851379394531, + "tps": 155.41790864306358, "num_tokens": 20, - "total_time": 0.3198511600494385, - "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" + "total_time": 0.4062361717224121, + "output": "Radiant orb descends, day's final scene,\nAmber and crimson blend in" }, { "model": "groq.com/mixtral-8x7b-32768", - "ttr": 0.17887043952941895, - "ttft": 0.2371835708618164, - "tps": 802.6971796937953, + "ttr": 0.1562638282775879, + "ttft": 0.24326205253601074, + "tps": 785.3031267552893, "num_tokens": 20, - "total_time": 0.26085376739501953, + "total_time": 0.2674565315246582, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { "model": "octoai.run/mixtral-8x7b-instruct", - "ttr": 0.14362049102783203, - "ttft": 0.17533636093139648, - "tps": 46.97170738262787, + "ttr": 0.16166305541992188, + "ttft": 0.5868933200836182, + "tps": 42.11727250626275, "num_tokens": 20, - "total_time": 0.5798351764678955, - "output": " Radiant sun, your day is done,\nDescending low, the race is won.\n" + "total_time": 1.0380146503448486, + "output": " Radiant sun descends, day's final bow,\nNine clouds in hues of" }, { "model": "perplexity.ai/mixtral-8x7b-instruct", - "ttr": 0.12900900840759277, - "ttft": 0.1293046474456787, - "tps": 153.15151072555568, + "ttr": 0.12902212142944336, + "ttft": 0.12923479080200195, + "tps": 152.55810249440052, "num_tokens": 20, - "total_time": 0.25336480140686035, + "total_time": 0.25377750396728516, "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." }, { "model": "perplexity.ai/sonar-medium-chat", - "ttr": 0.1454944610595703, - "ttft": 0.14555978775024414, - "tps": 65.52770126151786, + "ttr": 0.14195013046264648, + "ttft": 0.14549994468688965, + "tps": 46.29555296072314, "num_tokens": 20, - "total_time": 0.4355134963989258, + "total_time": 0.5559065341949463, "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" }, { - "model": "fireworks.ai/llama-v3-70b-instruct", - "ttr": 0.3684380054473877, - "ttft": 0.38796567916870117, - "tps": 98.62086820070738, + "model": "anyscale.com/Llama-3-70b-chat-hf", + "ttr": 0.34551286697387695, + "ttft": 0.42008137702941895, + "tps": 16.61024284513425, "num_tokens": 20, - "total_time": 0.580622673034668, + "total_time": 1.5639538764953613, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, + { + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.23824167251586914, + "ttft": 0.23836851119995117, + "tps": 124.09840446239757, + "num_tokens": 20, + "total_time": 0.39147281646728516, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nDance with" + }, { "model": "groq.com/llama3-70b-8192", - "ttr": 0.16277265548706055, - "ttft": 3.157120943069458, - "tps": 407.86422912359, + "ttr": 0.15274596214294434, + "ttft": 0.24539923667907715, + "tps": 636.597137014315, "num_tokens": 20, - "total_time": 3.203705072402954, + "total_time": 0.27524542808532715, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" }, { "model": "perplexity.ai/llama-3-70b-instruct", - "ttr": 0.17601966857910156, - "ttft": 0.17620062828063965, - "tps": 41.48575777625885, + "ttr": 0.16309475898742676, + "ttft": 0.16971158981323242, + "tps": 33.074565317179214, "num_tokens": 20, - "total_time": 0.6341891288757324, + "total_time": 0.744171142578125, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "together.xyz/Llama-3-70b-chat-hf", - "ttr": 0.29582691192626953, - "ttft": 0.2958822250366211, - "tps": 102.94472447495356, + "ttr": 0.3632166385650635, + "ttft": 0.3643643856048584, + "tps": 176.31313967123165, "num_tokens": 20, - "total_time": 0.48044729232788086, + "total_time": 0.47212719917297363, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "azure.westus3/llama-2-70b", - "ttr": 7.5272979736328125, - "ttft": 7.527413606643677, + "ttr": 1.2712457180023193, + "ttft": 1.2713069915771484, "tps": 999, "num_tokens": 20, - "total_time": 7.527679443359375, + "total_time": 1.2715365886688232, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "azure.eastus2/llama-2-70b", - "ttr": 1.4821009635925293, - "ttft": 1.4822006225585938, + "ttr": 1.4783165454864502, + "ttft": 1.478445053100586, "tps": 999, "num_tokens": 20, - "total_time": 1.482433557510376, + "total_time": 1.47877836227417, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" + }, + { + "model": "anyscale.com/Llama-2-70b-chat-hf", + "ttr": 0.42195868492126465, + "ttft": 0.4220433235168457, + "tps": 21.16844784201661, + "num_tokens": 20, + "total_time": 1.3196055889129639, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-70b-chat", - "ttr": 0.1928112506866455, - "ttft": 0.2099165916442871, - "tps": 112.6621201325507, + "ttr": 0.18727445602416992, + "ttft": 0.22904658317565918, + "tps": 130.79320395998005, "num_tokens": 20, - "total_time": 0.37856245040893555, + "total_time": 0.3743140697479248, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { "model": "groq.com/llama2-70b-4096", - "ttr": 0.15791654586791992, - "ttft": 0.3874807357788086, - "tps": 364.5493222777032, + "ttr": 0.16466045379638672, + "ttft": 0.35110998153686523, + "tps": 627.5187576208679, "num_tokens": 21, - "total_time": 0.44234299659729004, + "total_time": 0.382981538772583, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { "model": "octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.09893131256103516, - "ttft": 0.1474466323852539, - "tps": 28.342643506997657, + "ttr": 0.11053609848022461, + "ttft": 0.14747023582458496, + "tps": 27.265279198584935, "num_tokens": 20, - "total_time": 0.8178145885467529, + "total_time": 0.8443272113800049, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "together.xyz/llama-2-70b-chat", - "ttr": 0.29886674880981445, - "ttft": 0.2990248203277588, - "tps": 78.44975940903613, + "ttr": 0.3536818027496338, + "ttft": 0.3537921905517578, + "tps": 72.79609326900065, "num_tokens": 20, - "total_time": 0.5412180423736572, + "total_time": 0.6147952079772949, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-13b-chat", - "ttr": 0.23177242279052734, - "ttft": 0.2420203685760498, - "tps": 182.20468933981138, + "ttr": 0.23432183265686035, + "ttft": 0.25012898445129395, + "tps": 294.5394657086678, "num_tokens": 20, - "total_time": 0.3462986946105957, + "total_time": 0.3146364688873291, "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, + { + "model": "anyscale.com/Llama-2-13b-chat-hf", + "ttr": 0.4848172664642334, + "ttft": 0.4957706928253174, + "tps": 30.748392005673413, + "num_tokens": 20, + "total_time": 1.1136891841888428, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" + }, { "model": "together.xyz/llama-2-13b-chat", - "ttr": 0.2873845100402832, - "ttft": 0.2879202365875244, - "tps": 43.789288595147404, + "ttr": 0.31820034980773926, + "ttft": 0.3190743923187256, + "tps": 52.88517821224342, "num_tokens": 20, - "total_time": 0.7218163013458252, + "total_time": 0.6783432960510254, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { "model": "octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.10034346580505371, - "ttft": 0.12205886840820312, - "tps": 48.04069352119163, + "ttr": 0.09300780296325684, + "ttft": 0.11617064476013184, + "tps": 52.10475640637535, "num_tokens": 20, - "total_time": 0.51755690574646, + "total_time": 0.4808206558227539, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, + { + "model": "anyscale.com/Llama-3-8b-chat-hf", + "ttr": 0.46367502212524414, + "ttft": 0.46446776390075684, + "tps": 18.28150690949399, + "num_tokens": 20, + "total_time": 1.5037693977355957, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" + }, { "model": "fireworks.ai/llama-v3-8b-instruct", - "ttr": 0.22245573997497559, - "ttft": 0.2225174903869629, - "tps": 297.2553302597616, + "ttr": 0.22959637641906738, + "ttft": 0.23851370811462402, + "tps": 465.69880144691246, "num_tokens": 20, - "total_time": 0.286435604095459, - "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" + "total_time": 0.2793126106262207, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fading, wild" }, { "model": "groq.com/llama3-8b-8192", - "ttr": 0.16103720664978027, - "ttft": 4.76821494102478, - "tps": 999, + "ttr": 0.1608898639678955, + "ttft": 0.24776530265808105, + "tps": 475.67240278150837, "num_tokens": 20, - "total_time": 4.783697605133057, + "total_time": 0.28770875930786133, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "perplexity.ai/llama-3-8b-instruct", - "ttr": 0.12770795822143555, - "ttft": 0.12789368629455566, - "tps": 140.02631430981165, + "ttr": 0.14670681953430176, + "ttft": 0.1527388095855713, + "tps": 119.71214445053666, "num_tokens": 20, - "total_time": 0.2635824680328369, + "total_time": 0.31145286560058594, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "together.xyz/Llama-3-8b-chat-hf", - "ttr": 0.27105069160461426, - "ttft": 0.27110719680786133, - "tps": 205.22244855389513, + "ttr": 0.2860441207885742, + "ttft": 0.2950911521911621, + "tps": 143.9136934376168, "num_tokens": 20, - "total_time": 0.363689661026001, + "total_time": 0.42711472511291504, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, + { + "model": "anyscale.com/Llama-2-7b-chat-hf", + "ttr": 0.6781177520751953, + "ttft": 0.7061538696289062, + "tps": 16.02422582133135, + "num_tokens": 20, + "total_time": 1.8918585777282715, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" + }, { "model": "fireworks.ai/llama-v2-7b-chat", - "ttr": 0.24669766426086426, - "ttft": 0.25202465057373047, - "tps": 179.67861040214285, + "ttr": 0.22578048706054688, + "ttft": 0.22945499420166016, + "tps": 209.39999789790105, "num_tokens": 20, - "total_time": 0.3577690124511719, + "total_time": 0.3201904296875, "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "together.xyz/llama-2-7b-chat", - "ttr": 0.29149770736694336, - "ttft": 0.2920844554901123, - "tps": 101.12810204713539, + "ttr": 0.29892992973327637, + "ttft": 0.29898738861083984, + "tps": 98.71419971311708, "num_tokens": 20, - "total_time": 0.4799649715423584, + "total_time": 0.49146223068237305, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-fp16", - "ttr": 0.42386579513549805, - "ttft": 0.42392396926879883, - "tps": 19.483144167604213, + "ttr": 0.925818681716919, + "ttft": 0.9272687435150146, + "tps": 19.68914621769529, "num_tokens": 21, - "total_time": 1.4504523277282715, + "total_time": 1.9430568218231201, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-int8", - "ttr": 0.3685901165008545, - "ttft": 0.36864519119262695, - "tps": 36.242228121982464, + "ttr": 0.35996150970458984, + "ttft": 0.360400915145874, + "tps": 37.17397248315039, "num_tokens": 21, - "total_time": 0.920487642288208, + "total_time": 0.898411750793457, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ] }, { - "time": "2024-04-22T08:06:20.506448", + "time": "2024-04-22T21:48:18.166585", "region": "sea", "cmd": "--max-tokens=20", "results": [ { "model": "openai/gpt-4-turbo", - "ttr": 0.5042407512664795, - "ttft": 0.504305362701416, - "tps": 23.47526167896962, + "ttr": 0.5468933582305908, + "ttft": 0.5469613075256348, + "tps": 5.860270698909767, "num_tokens": 20, - "total_time": 1.3136680126190186, + "total_time": 3.7891323566436768, "output": "Sunset paints the sky with grace,\nHorizon blushes in embrace,\nColors blend in soft ballet" }, { "model": "openai/gpt-4-0125-preview", - "ttr": 0.30695271492004395, - "ttft": 0.3646430969238281, - "tps": 24.450580185929493, + "ttr": 0.37444567680358887, + "ttft": 0.45653653144836426, + "tps": 7.8602192408828815, "num_tokens": 20, - "total_time": 1.1417207717895508, + "total_time": 2.873771905899048, "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { "model": "azure.scentralus/gpt-4-0125-preview", - "ttr": 0.49054574966430664, - "ttft": 0.49062681198120117, - "tps": 11.871399071819992, + "ttr": 0.34078288078308105, + "ttft": 0.34084033966064453, + "tps": 22.32078814797885, "num_tokens": 20, - "total_time": 2.0911121368408203, - "output": "Beneath the vast, ever-changing sky,\nColors blend where the sun bids goodbye,\nOranges" + "total_time": 1.1920647621154785, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { "model": "openai/gpt-4-1106-preview", - "ttr": 0.3850069046020508, - "ttft": 0.4440650939941406, - "tps": 26.919623289059437, + "ttr": 0.5306136608123779, + "ttft": 0.9630041122436523, + "tps": 19.067581813631403, "num_tokens": 20, - "total_time": 1.1498699188232422, + "total_time": 1.9594597816467285, "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "azure.westus/gpt-4-1106-preview", - "ttr": 0.3797454833984375, - "ttft": 0.3798031806945801, - "tps": 10.190510083188462, + "ttr": 0.3344874382019043, + "ttft": 0.33457136154174805, + "tps": 22.278337993070917, "num_tokens": 20, - "total_time": 2.2442829608917236, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 1.18741774559021, + "output": "The sun dips low, horizon kissed,\nPainting skies with amethyst,\nCrimson streaks" }, { "model": "azure.eastus2/gpt-4-1106-preview", - "ttr": 0.4943256378173828, - "ttft": 0.4943814277648926, - "tps": 6.786844766628102, + "ttr": 0.5358996391296387, + "ttft": 0.5360383987426758, + "tps": 6.717119321077807, "num_tokens": 20, - "total_time": 3.293915033340454, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 3.3646318912506104, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "azure.frcentral/gpt-4-1106-preview", - "ttr": 1.2634272575378418, - "ttft": 1.263505458831787, - "tps": 8.617590999581513, + "ttr": 0.4289724826812744, + "ttft": 0.42903971672058105, + "tps": 18.23600836883467, "num_tokens": 20, - "total_time": 3.4682979583740234, - "output": "The sun dips low, horizon kissed,\nPainting skies with amethyst,\nCrimson streaks" + "total_time": 1.4709343910217285, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "azure.secentral/gpt-4-1106-preview", - "ttr": 0.9889154434204102, - "ttft": 0.9890758991241455, - "tps": 3.9578369882291615, + "ttr": 0.4176502227783203, + "ttft": 0.4177286624908447, + "tps": 22.264066223574407, "num_tokens": 20, - "total_time": 5.789677858352661, + "total_time": 1.2711217403411865, "output": "Golden sun dips beneath the horizon,\nPainting the sky with hues of passion,\nCrimson," }, { "model": "azure.uksouth/gpt-4-1106-preview", - "ttr": 0.8197438716888428, - "ttft": 0.8198344707489014, - "tps": 4.412255976234034, + "ttr": 0.8496823310852051, + "ttft": 0.8498983383178711, + "tps": 17.58252139078197, "num_tokens": 20, - "total_time": 5.12602162361145, + "total_time": 1.9305169582366943, "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { "model": "openai/gpt-3.5-turbo-0125", - "ttr": 0.2534823417663574, - "ttft": 0.27307987213134766, - "tps": 33.926350510267504, + "ttr": 0.2699756622314453, + "ttft": 0.30794191360473633, + "tps": 71.92710153508455, "num_tokens": 20, - "total_time": 0.8331165313720703, + "total_time": 0.5720982551574707, "output": "The sun dips below the horizon line\nPainting the sky in hues divine\nOrange, pink," }, { "model": "openai/gpt-3.5-turbo-1106", - "ttr": 0.381000280380249, - "ttft": 0.5620617866516113, - "tps": 332.3620409218681, + "ttr": 0.22860145568847656, + "ttft": 0.25156211853027344, + "tps": 72.18202980148347, "num_tokens": 20, - "total_time": 0.6192283630371094, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" + "total_time": 0.5147855281829834, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { "model": "azure.westus/gpt-3.5-turbo-1106", - "ttr": 0.0831611156463623, - "ttft": 0.08329010009765625, - "tps": 72.19085717598632, + "ttr": 0.06739401817321777, + "ttft": 0.0675044059753418, + "tps": 129.10589558029935, "num_tokens": 20, - "total_time": 0.3464813232421875, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" + "total_time": 0.21467041969299316, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" }, { "model": "azure.eastus2/gpt-3.5-turbo", - "ttr": 0.23638224601745605, - "ttft": 0.23682498931884766, - "tps": 54.292460131459904, + "ttr": 0.1307520866394043, + "ttft": 0.13229012489318848, + "tps": 67.73441628921586, "num_tokens": 20, - "total_time": 0.5867815017700195, - "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, bidding farewell.\nCrim" + "total_time": 0.4127974510192871, + "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" }, { "model": "anthropic/claude-3-opus-20240229", - "ttr": 0.8946642875671387, - "ttft": 0.8947696685791016, - "tps": 39.140959322599976, + "ttr": 0.9233033657073975, + "ttft": 0.9233756065368652, + "tps": 39.25794144002349, "num_tokens": 20, - "total_time": 1.3801946640014648, + "total_time": 1.4073541164398193, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { "model": "anthropic/claude-3-sonnet-20240229", - "ttr": 0.6642239093780518, - "ttft": 0.6666467189788818, - "tps": 71.55118323033508, + "ttr": 0.6195611953735352, + "ttft": 0.6197924613952637, + "tps": 76.18512697556854, "num_tokens": 20, - "total_time": 0.9321908950805664, + "total_time": 0.8691849708557129, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { "model": "anthropic/claude-3-haiku-20240307", - "ttr": 0.281796932220459, - "ttft": 0.28218507766723633, - "tps": 99.60625346845518, + "ttr": 0.30507349967956543, + "ttft": 0.3052072525024414, + "tps": 125.05967390110321, "num_tokens": 20, - "total_time": 0.47293615341186523, - "output": "Here is a nonet about a sunset:\n\nSunset's golden glow\nPaints" + "total_time": 0.4571347236633301, + "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," }, { "model": "anthropic/claude-2.1", - "ttr": 0.41651487350463867, - "ttft": 0.41733622550964355, - "tps": 32.08855906696007, + "ttr": 0.33652186393737793, + "ttft": 0.33969950675964355, + "tps": 19.354840715323814, "num_tokens": 20, - "total_time": 1.0094475746154785, - "output": "Here is a nonet poem about a sunset:\n\nSun descending\nGolden horizon glowing\nCl" + "total_time": 1.3213660717010498, + "output": "Here is a nonet poem about a sunset:\n\nRadiant, glowing\nColors fill the" }, { "model": "anthropic/claude-instant-1.2", - "ttr": 0.4013829231262207, - "ttft": 0.40471959114074707, - "tps": 170.99551546413872, + "ttr": 0.41101694107055664, + "ttft": 0.41237473487854004, + "tps": 169.4506129131715, "num_tokens": 20, - "total_time": 0.5158336162567139, + "total_time": 0.5245018005371094, "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" }, { "model": "cohere/command-r-plus", - "ttr": 0.06077742576599121, - "ttft": 0.5973734855651855, - "tps": 31.709284584660875, + "ttr": 0.0695650577545166, + "ttft": 0.24341106414794922, + "tps": 23.442089095419057, "num_tokens": 17, - "total_time": 1.1019575595855713, + "total_time": 0.9259440898895264, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { "model": "cohere/command-r", - "ttr": 0.07931971549987793, - "ttft": 0.1299593448638916, - "tps": 92.57626880980511, + "ttr": 0.08441805839538574, + "ttft": 0.15618109703063965, + "tps": 84.53942053351133, "num_tokens": 18, - "total_time": 0.31359171867370605, - "output": "The horizon's fiery blaze,\nThe sky a canvas of amber and red,\nThe sun slowly" + "total_time": 0.3572707176208496, + "output": "The horizon's hue, a golden glow,\nThe sky's fiery embrace,\nClouds in" }, { "model": "cohere/command-light", - "ttr": 0.08011507987976074, - "ttft": 0.1263134479522705, - "tps": 82.85998098837223, + "ttr": 0.08730030059814453, + "ttft": 0.15821409225463867, + "tps": 76.90060956933257, "num_tokens": 18, - "total_time": 0.3314788341522217, + "total_time": 0.37927865982055664, "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { "model": "azure.eastus2/mistral", - "ttr": 0.9781889915466309, - "ttft": 0.9782519340515137, + "ttr": 1.0192115306854248, + "ttft": 1.0192737579345703, "tps": 999, "num_tokens": 20, - "total_time": 0.9783940315246582, + "total_time": 1.0194172859191895, "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" }, + { + "model": "anyscale.com/Mixtral-8x7B-Instruct-v0.1", + "ttr": 0.34307193756103516, + "ttft": 0.4029526710510254, + "tps": 16.865327572428924, + "num_tokens": 20, + "total_time": 1.529524326324463, + "output": " Sunset's glow, soon to wane,\nDay's fiery orb in night" + }, { "model": "fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.13983511924743652, - "ttft": 0.143540620803833, - "tps": 199.86902086677367, + "ttr": 0.1727123260498047, + "ttft": 0.1763153076171875, + "tps": 183.4103014959724, "num_tokens": 20, - "total_time": 0.238602876663208, - "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" + "total_time": 0.2799081802368164, + "output": "Radiant orb descends, day's final scene,\nAmber and crimson blend in" }, { "model": "groq.com/mixtral-8x7b-32768", - "ttr": 0.034388065338134766, - "ttft": 0.11794114112854004, - "tps": 729.4775596137123, + "ttr": 0.04238748550415039, + "ttft": 0.1282031536102295, + "tps": 999, "num_tokens": 20, - "total_time": 0.14398717880249023, + "total_time": 0.1383516788482666, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { "model": "octoai.run/mixtral-8x7b-instruct", - "ttr": 0.25011777877807617, - "ttft": 0.27352166175842285, - "tps": 56.69756730240424, + "ttr": 0.2500898838043213, + "ttft": 0.27245664596557617, + "tps": 51.947499581183024, "num_tokens": 20, - "total_time": 0.6086330413818359, + "total_time": 0.6382105350494385, "output": " Radiant sun descends, a fiery sphere,\nIts brilliant hues of red and" }, { "model": "perplexity.ai/mixtral-8x7b-instruct", - "ttr": 0.13250970840454102, - "ttft": 0.13263940811157227, - "tps": 152.36559279582434, + "ttr": 0.12467789649963379, + "ttft": 0.12531757354736328, + "tps": 128.09649538756804, "num_tokens": 20, - "total_time": 0.2573394775390625, + "total_time": 0.27364325523376465, "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." }, { "model": "perplexity.ai/sonar-medium-chat", - "ttr": 0.142625093460083, - "ttft": 0.14266490936279297, - "tps": 75.10567337252677, + "ttr": 0.16072440147399902, + "ttft": 0.16084957122802734, + "tps": 74.5476873800285, "num_tokens": 20, - "total_time": 0.3956418037414551, - "output": "Radiant sun, descending low,\nCasting hues of crimson glow.\nS" + "total_time": 0.41571998596191406, + "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" }, { - "model": "fireworks.ai/llama-v3-70b-instruct", - "ttr": 0.21239638328552246, - "ttft": 0.2136092185974121, - "tps": 66.53862101384597, + "model": "anyscale.com/Llama-3-70b-chat-hf", + "ttr": 0.2804439067840576, + "ttft": 0.28245043754577637, + "tps": 17.131517420058284, "num_tokens": 20, - "total_time": 0.4991576671600342, + "total_time": 1.3915174007415771, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, + { + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.17885947227478027, + "ttft": 0.1789252758026123, + "tps": 153.20539138693064, + "num_tokens": 20, + "total_time": 0.30294179916381836, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nDance with" + }, { "model": "groq.com/llama3-70b-8192", - "ttr": 0.04072880744934082, - "ttft": 0.1012113094329834, - "tps": 281.57847204064757, + "ttr": 0.03165841102600098, + "ttft": 0.13626456260681152, + "tps": 868.9919525439993, "num_tokens": 20, - "total_time": 0.16868805885314941, + "total_time": 0.15812897682189941, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" }, { "model": "perplexity.ai/llama-3-70b-instruct", - "ttr": 0.18464970588684082, - "ttft": 0.18507742881774902, - "tps": 38.33196696472807, + "ttr": 0.18588852882385254, + "ttft": 0.18601608276367188, + "tps": 42.55727588238027, "num_tokens": 20, - "total_time": 0.6807472705841064, + "total_time": 0.6324732303619385, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "together.xyz/Llama-3-70b-chat-hf", - "ttr": 0.17232275009155273, - "ttft": 0.17242932319641113, - "tps": 100.58866979529293, + "ttr": 0.17174410820007324, + "ttft": 0.17179298400878906, + "tps": 102.33372712983984, "num_tokens": 20, - "total_time": 0.36131739616394043, + "total_time": 0.35746002197265625, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "azure.westus3/llama-2-70b", - "ttr": 7.415329933166504, - "ttft": 7.415436267852783, + "ttr": 1.8263030052185059, + "ttft": 1.826432228088379, "tps": 999, "num_tokens": 20, - "total_time": 7.41561484336853, + "total_time": 1.8266334533691406, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "azure.eastus2/llama-2-70b", - "ttr": 1.938852310180664, - "ttft": 1.9389588832855225, + "ttr": 1.7574000358581543, + "ttft": 1.7575557231903076, "tps": 999, "num_tokens": 20, - "total_time": 1.9391627311706543, + "total_time": 1.7578413486480713, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" + }, + { + "model": "anyscale.com/Llama-2-70b-chat-hf", + "ttr": 1.270235538482666, + "ttft": 1.4231257438659668, + "tps": 33.2029698267601, + "num_tokens": 20, + "total_time": 1.995363712310791, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-70b-chat", - "ttr": 0.1957550048828125, - "ttft": 0.21876811981201172, - "tps": 188.54729310891446, + "ttr": 0.2498912811279297, + "ttft": 0.2745816707611084, + "tps": 124.68224025633684, "num_tokens": 20, - "total_time": 0.31953859329223633, + "total_time": 0.426969051361084, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { "model": "groq.com/llama2-70b-4096", - "ttr": 0.023345470428466797, - "ttft": 0.2662339210510254, - "tps": 454.4725022889928, + "ttr": 0.04070281982421875, + "ttft": 0.21325325965881348, + "tps": 598.4211615149202, "num_tokens": 21, - "total_time": 0.3102409839630127, + "total_time": 0.2466745376586914, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { "model": "octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.2037487030029297, - "ttft": 0.2511575222015381, - "tps": 26.908543044099112, + "ttr": 0.19350433349609375, + "ttft": 0.23502588272094727, + "tps": 26.334673224688014, "num_tokens": 20, - "total_time": 0.9572529792785645, + "total_time": 0.9565081596374512, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "together.xyz/llama-2-70b-chat", - "ttr": 0.21956515312194824, - "ttft": 0.21961188316345215, - "tps": 74.50217780092123, + "ttr": 0.1753249168395996, + "ttft": 0.17644047737121582, + "tps": 75.36072005757131, "num_tokens": 20, - "total_time": 0.4746379852294922, + "total_time": 0.4285612106323242, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-13b-chat", - "ttr": 0.18693780899047852, - "ttft": 0.19986724853515625, - "tps": 197.511093486666, + "ttr": 0.22391581535339355, + "ttft": 0.2342386245727539, + "tps": 70.53898942691114, "num_tokens": 20, - "total_time": 0.2960643768310547, + "total_time": 0.5035932064056396, "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, + { + "model": "anyscale.com/Llama-2-13b-chat-hf", + "ttr": 0.4496753215789795, + "ttft": 0.45130252838134766, + "tps": 38.339195938801055, + "num_tokens": 20, + "total_time": 0.9468789100646973, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" + }, { "model": "together.xyz/llama-2-13b-chat", - "ttr": 0.1624457836151123, - "ttft": 0.1627030372619629, - "tps": 42.29470937829384, + "ttr": 0.1764392852783203, + "ttft": 0.1770341396331787, + "tps": 40.43350817429706, "num_tokens": 20, - "total_time": 0.6119318008422852, + "total_time": 0.6469414234161377, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { "model": "octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.0747978687286377, - "ttft": 0.09755325317382812, - "tps": 52.313744179741526, + "ttr": 0.07695245742797852, + "ttft": 0.10211181640625, + "tps": 53.39092555384565, "num_tokens": 20, - "total_time": 0.46074652671813965, + "total_time": 0.4579775333404541, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, + { + "model": "anyscale.com/Llama-3-8b-chat-hf", + "ttr": 0.8496425151824951, + "ttft": 0.8558862209320068, + "tps": 24.24543168319603, + "num_tokens": 20, + "total_time": 1.6395390033721924, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" + }, { "model": "fireworks.ai/llama-v3-8b-instruct", - "ttr": 0.17707538604736328, - "ttft": 0.18795156478881836, - "tps": 357.7408198811298, + "ttr": 0.18891191482543945, + "ttft": 0.18896245956420898, + "tps": 369.10916476380595, "num_tokens": 20, - "total_time": 0.24106264114379883, - "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" + "total_time": 0.24043774604797363, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fading, wild" }, { "model": "groq.com/llama3-8b-8192", - "ttr": 0.04481387138366699, - "ttft": 0.15939688682556152, - "tps": 789.4104664639281, + "ttr": 0.030283212661743164, + "ttft": 4.798012971878052, + "tps": 717.9698007135393, "num_tokens": 20, - "total_time": 0.18346548080444336, + "total_time": 4.824476480484009, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "perplexity.ai/llama-3-8b-instruct", - "ttr": 0.1204688549041748, - "ttft": 0.12056541442871094, - "tps": 129.21516951309877, + "ttr": 0.13428878784179688, + "ttft": 0.13735055923461914, + "tps": 118.52898236011542, "num_tokens": 20, - "total_time": 0.2676069736480713, + "total_time": 0.29764890670776367, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "together.xyz/Llama-3-8b-chat-hf", - "ttr": 0.1269392967224121, - "ttft": 0.12736988067626953, - "tps": 217.38369590419947, + "ttr": 0.13915324211120605, + "ttft": 0.13919353485107422, + "tps": 208.56097963114658, "num_tokens": 20, - "total_time": 0.21477293968200684, + "total_time": 0.23029398918151855, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, + { + "model": "anyscale.com/Llama-2-7b-chat-hf", + "ttr": 0.7751650810241699, + "ttft": 0.7880799770355225, + "tps": 14.212796617968399, + "num_tokens": 20, + "total_time": 2.124903440475464, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" + }, { "model": "fireworks.ai/llama-v2-7b-chat", - "ttr": 0.16161346435546875, - "ttft": 0.17016339302062988, - "tps": 280.83326931412523, + "ttr": 0.21126651763916016, + "ttft": 0.21654677391052246, + "tps": 228.59930753365518, "num_tokens": 20, - "total_time": 0.23781919479370117, + "total_time": 0.29966163635253906, "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "together.xyz/llama-2-7b-chat", - "ttr": 0.14069080352783203, - "ttft": 0.14288878440856934, - "tps": 102.78804388483455, + "ttr": 0.1639881134033203, + "ttft": 0.16402769088745117, + "tps": 98.45747544795361, "num_tokens": 20, - "total_time": 0.32773518562316895, + "total_time": 0.35700440406799316, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-fp16", - "ttr": 0.5737338066101074, - "ttft": 0.5739378929138184, - "tps": 19.684526005250703, + "ttr": 0.6264975070953369, + "ttft": 0.6265466213226318, + "tps": 19.29965816221121, "num_tokens": 21, - "total_time": 1.5899643898010254, + "total_time": 1.6628344058990479, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-int8", - "ttr": 1.0605113506317139, - "ttft": 1.0606534481048584, - "tps": 36.519859172772826, + "ttr": 0.5942039489746094, + "ttft": 0.5942685604095459, + "tps": 29.940900631825713, "num_tokens": 21, - "total_time": 1.6083006858825684, + "total_time": 1.2622511386871338, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ] }, { - "time": "2024-04-22T08:06:42.176632", + "time": "2024-04-22T21:48:29.239594", "region": "iad", "cmd": "--max-tokens=20", "results": [ { "model": "openai/gpt-4-turbo", - "ttr": 0.4815518856048584, - "ttft": 0.4816761016845703, - "tps": 26.0290968049171, + "ttr": 0.5559380054473877, + "ttft": 0.5562891960144043, + "tps": 48.591272302006715, "num_tokens": 20, - "total_time": 1.2116284370422363, - "output": "Sunset paints the sky with grace,\nHorizon's blush at day's embrace,\nColors meld in" + "total_time": 0.9473059177398682, + "output": "Golden blaze dips low, kissing sea,\nHorizon melts in fiery hues,\nCrimson, orange" }, { "model": "openai/gpt-4-0125-preview", - "ttr": 0.4113337993621826, - "ttft": 0.4861607551574707, - "tps": 16.533285643954947, + "ttr": 0.335188627243042, + "ttft": 0.3870093822479248, + "tps": 34.44773799381087, "num_tokens": 20, - "total_time": 1.6353576183319092, - "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye,\n" + "total_time": 0.9385695457458496, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { "model": "azure.scentralus/gpt-4-0125-preview", - "ttr": 0.5049262046813965, - "ttft": 0.5050053596496582, - "tps": 9.412204010555984, + "ttr": 0.34845447540283203, + "ttft": 0.34859657287597656, + "tps": 10.225830161330721, "num_tokens": 20, - "total_time": 2.5236611366271973, - "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" + "total_time": 2.206636428833008, + "output": "Amidst the quiet of the evening sky,\nA canvas painted with a golden hue,\nWhere orange" }, { "model": "openai/gpt-4-1106-preview", - "ttr": 0.4584345817565918, - "ttft": 0.5304193496704102, - "tps": 25.981498090635554, + "ttr": 0.5632400512695312, + "ttft": 0.6232304573059082, + "tps": 12.45817420294631, "num_tokens": 20, - "total_time": 1.2617089748382568, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 2.1483335494995117, + "output": "Amber hues dance across the evening sky,\nHorizon ablaze with the day's last sigh,\n" }, { "model": "azure.westus/gpt-4-1106-preview", - "ttr": 0.5812273025512695, - "ttft": 0.5815980434417725, - "tps": 17.205045722215843, + "ttr": 0.4613916873931885, + "ttft": 0.46154284477233887, + "tps": 23.477225771992874, "num_tokens": 20, - "total_time": 1.6859252452850342, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 1.2708377838134766, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { "model": "azure.eastus2/gpt-4-1106-preview", - "ttr": 0.4476635456085205, - "ttft": 0.44777560234069824, - "tps": 14.149077063612612, + "ttr": 0.35037946701049805, + "ttft": 0.3736104965209961, + "tps": 12.813068427560104, "num_tokens": 20, - "total_time": 1.7906193733215332, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 1.8564715385437012, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "azure.frcentral/gpt-4-1106-preview", - "ttr": 0.6240036487579346, - "ttft": 0.6242091655731201, - "tps": 11.061335117394808, + "ttr": 0.55851149559021, + "ttft": 0.5586895942687988, + "tps": 22.688121158528894, "num_tokens": 20, - "total_time": 2.3419041633605957, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with hues of gold and pink" + "total_time": 1.396132230758667, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { "model": "azure.secentral/gpt-4-1106-preview", - "ttr": 0.5716967582702637, - "ttft": 0.5718288421630859, - "tps": 14.748447505561774, + "ttr": 0.6419315338134766, + "ttft": 0.6420488357543945, + "tps": 30.3219234530228, "num_tokens": 20, - "total_time": 1.8601000308990479, - "output": "Golden sun dips low, painting sky,\nCrimson, orange, pink hues comply,\nClouds" + "total_time": 1.26865816116333, + "output": "Golden orb descends in the sky,\nPainting hues of red and gold,\nSilhouettes of" }, { "model": "azure.uksouth/gpt-4-1106-preview", - "ttr": 0.6359684467315674, - "ttft": 0.6361269950866699, - "tps": 8.847335844444899, + "ttr": 0.565222978591919, + "ttft": 0.5653748512268066, + "tps": 8.982016755387793, "num_tokens": 20, - "total_time": 2.7836661338806152, + "total_time": 2.6807126998901367, "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { "model": "openai/gpt-3.5-turbo-0125", - "ttr": 0.35339999198913574, - "ttft": 0.38275861740112305, - "tps": 52.58968291153859, + "ttr": 0.33823299407958984, + "ttft": 0.39188408851623535, + "tps": 68.00747903455256, "num_tokens": 20, - "total_time": 0.7440462112426758, + "total_time": 0.6712651252746582, "output": "The sun dips below the horizon line\nPainting the sky in hues divine\nOrange, pink," }, { "model": "openai/gpt-3.5-turbo-1106", - "ttr": 0.3066542148590088, - "ttft": 0.3278391361236572, - "tps": 79.97127560825083, + "ttr": 0.1852250099182129, + "ttft": 0.19980859756469727, + "tps": 107.5198480265306, "num_tokens": 20, - "total_time": 0.5654244422912598, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" + "total_time": 0.37652015686035156, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" }, { "model": "azure.westus/gpt-3.5-turbo-1106", - "ttr": 0.13196444511413574, - "ttft": 0.1583099365234375, - "tps": 216.86784391523594, + "ttr": 0.15768122673034668, + "ttft": 0.19008660316467285, + "tps": 177.75923517720878, "num_tokens": 20, - "total_time": 0.24592089653015137, + "total_time": 0.29697275161743164, "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { "model": "azure.eastus2/gpt-3.5-turbo", - "ttr": 0.09393525123596191, - "ttft": 0.10218954086303711, - "tps": 67.47456192043784, + "ttr": 0.08371496200561523, + "ttft": 0.0840446949005127, + "tps": 53.32918612066756, "num_tokens": 20, - "total_time": 0.3837771415710449, + "total_time": 0.4403223991394043, "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" }, { "model": "anthropic/claude-3-opus-20240229", - "ttr": 0.9463369846343994, - "ttft": 0.9466469287872314, - "tps": 40.08392639690804, + "ttr": 1.1362149715423584, + "ttft": 1.1366426944732666, + "tps": 40.56081288796324, "num_tokens": 20, - "total_time": 1.4206523895263672, + "total_time": 1.6050751209259033, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { "model": "anthropic/claude-3-sonnet-20240229", - "ttr": 0.6284692287445068, - "ttft": 0.6394884586334229, - "tps": 83.13922719486216, + "ttr": 0.6384725570678711, + "ttft": 0.6399140357971191, + "tps": 74.64446646647022, "num_tokens": 20, - "total_time": 0.86802077293396, + "total_time": 0.8944540023803711, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { "model": "anthropic/claude-3-haiku-20240307", - "ttr": 0.26909780502319336, - "ttft": 0.2728614807128906, - "tps": 113.63678960764882, + "ttr": 0.27796459197998047, + "ttft": 0.2914395332336426, + "tps": 101.07256541541052, "num_tokens": 20, - "total_time": 0.4400608539581299, + "total_time": 0.47942328453063965, "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," }, { "model": "anthropic/claude-2.1", - "ttr": 0.39328551292419434, - "ttft": 0.4193079471588135, - "tps": 29.582680059067528, + "ttr": 0.4489905834197998, + "ttft": 0.45483851432800293, + "tps": 31.298182279473302, "num_tokens": 20, - "total_time": 1.0615756511688232, - "output": "Here is a nonet poem about a sunset:\n\nRadiant, glowing\nColors fill the" + "total_time": 1.0619025230407715, + "output": "Here is a nonet poem about a sunset:\n\nSun descending\nGolden horizon glowing\nCl" }, { "model": "anthropic/claude-instant-1.2", - "ttr": 0.4070167541503906, - "ttft": 0.4090762138366699, - "tps": 171.46386730726613, + "ttr": 0.500324010848999, + "ttft": 0.5018417835235596, + "tps": 154.8600888059774, "num_tokens": 20, - "total_time": 0.5198867321014404, - "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" + "total_time": 0.6245331764221191, + "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe setting sun sinks" }, { "model": "cohere/command-r-plus", - "ttr": 0.0662691593170166, - "ttft": 0.21483230590820312, - "tps": 33.492437498190846, + "ttr": 0.06934976577758789, + "ttft": 0.22824668884277344, + "tps": 25.54759771997821, "num_tokens": 17, - "total_time": 0.6925520896911621, + "total_time": 0.8545286655426025, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { "model": "cohere/command-r", - "ttr": 0.05870628356933594, - "ttft": 0.12244176864624023, - "tps": 90.47431303482789, + "ttr": 0.08059453964233398, + "ttft": 0.14727234840393066, + "tps": 96.50848099963726, "num_tokens": 18, - "total_time": 0.31034040451049805, - "output": "The horizon's fiery blaze,\nThe sky a canvas of amber and red,\nThe sun slowly" + "total_time": 0.3234226703643799, + "output": "The horizon's fiery blaze,\nThe sky's golden embrace,\nSun slowly sinks, a h" }, { "model": "cohere/command-light", - "ttr": 0.05734443664550781, - "ttft": 0.13303875923156738, - "tps": 80.71465862499123, + "ttr": 0.08772754669189453, + "ttft": 0.18198013305664062, + "tps": 103.68429027403137, "num_tokens": 18, - "total_time": 0.3436572551727295, + "total_time": 0.34593939781188965, "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { "model": "azure.eastus2/mistral", - "ttr": 0.7112693786621094, - "ttft": 0.7113533020019531, + "ttr": 0.8799281120300293, + "ttft": 0.8800783157348633, "tps": 999, "num_tokens": 20, - "total_time": 0.7115473747253418, + "total_time": 0.8803505897521973, "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" }, + { + "model": "anyscale.com/Mixtral-8x7B-Instruct-v0.1", + "ttr": 0.27995753288269043, + "ttft": 0.28006696701049805, + "tps": 25.52015115124732, + "num_tokens": 20, + "total_time": 1.0245766639709473, + "output": " Sunset's glow, soon to wane,\nDay's fiery orb in night" + }, { "model": "fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.14188146591186523, - "ttft": 0.1485135555267334, - "tps": 147.56506587414012, + "ttr": 2.4045498371124268, + "ttft": 2.5638155937194824, + "tps": 34.97123067113514, "num_tokens": 20, - "total_time": 0.2772703170776367, - "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" + "total_time": 3.10711932182312, + "output": "Radiant orb descends, day's final scene,\nAmber and crimson blend in" }, { "model": "groq.com/mixtral-8x7b-32768", - "ttr": 0.20771360397338867, - "ttft": 0.2905716896057129, - "tps": 999, + "ttr": 0.11244797706604004, + "ttft": 0.2111828327178955, + "tps": 264.90370404842554, "num_tokens": 20, - "total_time": 0.2909269332885742, + "total_time": 0.28290700912475586, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { "model": "octoai.run/mixtral-8x7b-instruct", - "ttr": 0.09477424621582031, - "ttft": 0.12925434112548828, - "tps": 57.35082102178477, + "ttr": 0.07327580451965332, + "ttft": 0.10663580894470215, + "tps": 45.27139279854935, "num_tokens": 20, - "total_time": 0.46054863929748535, - "output": " Radiant sun descends, a fiery sphere,\nIts brilliant hues of red and" + "total_time": 0.5263268947601318, + "output": " Radiant sun, your day is done,\nSinking low, the race is won.\n" }, { "model": "perplexity.ai/mixtral-8x7b-instruct", - "ttr": 0.06856942176818848, - "ttft": 0.08472752571105957, - "tps": 151.0615701030244, + "ttr": 0.15790438652038574, + "ttft": 0.18107199668884277, + "tps": 179.79901043934598, "num_tokens": 20, - "total_time": 0.21050405502319336, + "total_time": 0.286745548248291, "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." }, { "model": "perplexity.ai/sonar-medium-chat", - "ttr": 0.06742739677429199, - "ttft": 0.0834968090057373, - "tps": 75.50917194748492, + "ttr": 0.10184645652770996, + "ttft": 0.10332465171813965, + "tps": 68.91617993735504, "num_tokens": 20, - "total_time": 0.33512187004089355, + "total_time": 0.37902188301086426, "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" }, { - "model": "fireworks.ai/llama-v3-70b-instruct", - "ttr": 0.1935124397277832, - "ttft": 0.20126914978027344, - "tps": 45.54781385082209, + "model": "anyscale.com/Llama-3-70b-chat-hf", + "ttr": 0.15503430366516113, + "ttft": 0.15628290176391602, + "tps": 12.347000461549689, "num_tokens": 20, - "total_time": 0.6184132099151611, + "total_time": 1.6951181888580322, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, + { + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.18986010551452637, + "ttft": 0.2127094268798828, + "tps": 186.79190125493983, + "num_tokens": 20, + "total_time": 0.31442689895629883, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nDance with" + }, { "model": "groq.com/llama3-70b-8192", - "ttr": 0.11503291130065918, - "ttft": 0.23969507217407227, - "tps": 163.8827901199532, + "ttr": 0.1450052261352539, + "ttft": 0.22111773490905762, + "tps": 506.17235772357725, "num_tokens": 20, - "total_time": 0.35563158988952637, + "total_time": 0.2586543560028076, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" }, { "model": "perplexity.ai/llama-3-70b-instruct", - "ttr": 0.1144716739654541, - "ttft": 0.12107729911804199, - "tps": 41.7589708521648, + "ttr": 0.0862424373626709, + "ttft": 0.09306883811950684, + "tps": 47.138886555690554, "num_tokens": 20, - "total_time": 0.5760693550109863, + "total_time": 0.49613308906555176, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "together.xyz/Llama-3-70b-chat-hf", - "ttr": 0.2793278694152832, - "ttft": 0.2793872356414795, - "tps": 95.77221886390649, + "ttr": 0.30110669136047363, + "ttft": 0.30141305923461914, + "tps": 139.74467398604514, "num_tokens": 20, - "total_time": 0.47777462005615234, + "total_time": 0.4373753070831299, "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { "model": "azure.westus3/llama-2-70b", - "ttr": 1.2381916046142578, - "ttft": 1.2382867336273193, + "ttr": 1.3906266689300537, + "ttft": 1.390742540359497, "tps": 999, "num_tokens": 20, - "total_time": 1.2385425567626953, + "total_time": 1.391019344329834, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "azure.eastus2/llama-2-70b", - "ttr": 1.2713744640350342, - "ttft": 1.2716302871704102, + "ttr": 1.6191773414611816, + "ttft": 1.619307518005371, "tps": 999, "num_tokens": 20, - "total_time": 1.271965742111206, + "total_time": 1.6196906566619873, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" + }, + { + "model": "anyscale.com/Llama-2-70b-chat-hf", + "ttr": 0.31021952629089355, + "ttft": 0.3385641574859619, + "tps": 32.77735340594726, + "num_tokens": 20, + "total_time": 0.9182326793670654, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-70b-chat", - "ttr": 0.17180705070495605, - "ttft": 0.1923677921295166, - "tps": 113.18857714235394, + "ttr": 0.1469407081604004, + "ttft": 0.16198420524597168, + "tps": 127.0039061317184, "num_tokens": 20, - "total_time": 0.3602292537689209, + "total_time": 0.3115859031677246, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { "model": "groq.com/llama2-70b-4096", - "ttr": 0.13243913650512695, - "ttft": 0.3076162338256836, - "tps": 188.095924659454, + "ttr": 0.1271507740020752, + "ttft": 0.3164865970611572, + "tps": 826.748928201843, "num_tokens": 21, - "total_time": 0.41394495964050293, + "total_time": 0.34067773818969727, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { "model": "octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.015279293060302734, - "ttft": 0.0740976333618164, - "tps": 27.93512073459072, + "ttr": 0.027872085571289062, + "ttft": 0.06821012496948242, + "tps": 26.47891917647047, "num_tokens": 20, - "total_time": 0.7542450428009033, + "total_time": 0.7857620716094971, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "together.xyz/llama-2-70b-chat", - "ttr": 0.25617337226867676, - "ttft": 0.2562408447265625, - "tps": 73.89583043561312, + "ttr": 0.3872096538543701, + "ttft": 0.38730955123901367, + "tps": 73.91193647925847, "num_tokens": 20, - "total_time": 0.513359546661377, + "total_time": 0.6443722248077393, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { "model": "fireworks.ai/llama-v2-13b-chat", - "ttr": 0.17316627502441406, - "ttft": 0.1915881633758545, - "tps": 256.7480677472462, + "ttr": 0.17949557304382324, + "ttft": 0.2029411792755127, + "tps": 290.6199779003913, "num_tokens": 20, - "total_time": 0.2655906677246094, + "total_time": 0.26831865310668945, "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, + { + "model": "anyscale.com/Llama-2-13b-chat-hf", + "ttr": 0.16283106803894043, + "ttft": 0.1630856990814209, + "tps": 40.278397431620746, + "num_tokens": 20, + "total_time": 0.6348025798797607, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" + }, { "model": "together.xyz/llama-2-13b-chat", - "ttr": 0.23114252090454102, - "ttft": 0.23120808601379395, - "tps": 43.18192955904912, + "ttr": 0.22519350051879883, + "ttft": 0.22550463676452637, + "tps": 38.92933571981273, "num_tokens": 20, - "total_time": 0.6712069511413574, + "total_time": 0.7135684490203857, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { "model": "octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.046790361404418945, - "ttft": 0.05956387519836426, - "tps": 56.28220564264386, + "ttr": 0.024411439895629883, + "ttft": 0.06444311141967773, + "tps": 54.5862870850038, "num_tokens": 20, - "total_time": 0.39714837074279785, + "total_time": 0.41251587867736816, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, + { + "model": "anyscale.com/Llama-3-8b-chat-hf", + "ttr": 0.44001293182373047, + "ttft": 0.44382143020629883, + "tps": 25.677081725665072, + "num_tokens": 20, + "total_time": 1.1837809085845947, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" + }, { "model": "fireworks.ai/llama-v3-8b-instruct", - "ttr": 0.173828125, - "ttft": 0.1925356388092041, - "tps": 709.1656077028495, + "ttr": 0.17583012580871582, + "ttft": 0.20998477935791016, + "tps": 933.0169412150375, "num_tokens": 20, - "total_time": 0.21932768821716309, - "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" + "total_time": 0.23034882545471191, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fading, wild" }, { "model": "groq.com/llama3-8b-8192", - "ttr": 0.1824946403503418, - "ttft": 3.535099744796753, + "ttr": 0.14323949813842773, + "ttft": 0.21816492080688477, "tps": 999, "num_tokens": 20, - "total_time": 3.5455799102783203, + "total_time": 0.21865296363830566, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "perplexity.ai/llama-3-8b-instruct", - "ttr": 0.10976505279541016, - "ttft": 0.12586283683776855, - "tps": 151.55022402081227, + "ttr": 0.10463929176330566, + "ttft": 0.1326274871826172, + "tps": 140.41020219746144, "num_tokens": 20, - "total_time": 0.2512338161468506, + "total_time": 0.2679452896118164, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { "model": "together.xyz/Llama-3-8b-chat-hf", - "ttr": 0.21121788024902344, - "ttft": 0.22036385536193848, - "tps": 198.23923502106976, + "ttr": 0.23333239555358887, + "ttft": 0.242767333984375, + "tps": 242.39074866017387, "num_tokens": 20, - "total_time": 0.3162076473236084, + "total_time": 0.3211531639099121, "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, + { + "model": "anyscale.com/Llama-2-7b-chat-hf", + "ttr": 0.1639728546142578, + "ttft": 0.16408348083496094, + "tps": 24.492843171846243, + "num_tokens": 20, + "total_time": 0.9398202896118164, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" + }, { "model": "fireworks.ai/llama-v2-7b-chat", - "ttr": 0.16304779052734375, - "ttft": 0.17381930351257324, - "tps": 269.73837754407816, + "ttr": 0.19978117942810059, + "ttft": 0.2214221954345703, + "tps": 908.7586922559383, "num_tokens": 20, - "total_time": 0.24425792694091797, + "total_time": 0.24232983589172363, "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "together.xyz/llama-2-7b-chat", - "ttr": 0.21961617469787598, - "ttft": 0.21973204612731934, - "tps": 88.61563643601613, + "ttr": 0.3376128673553467, + "ttft": 0.3411712646484375, + "tps": 112.81922286856549, "num_tokens": 20, - "total_time": 0.4341411590576172, + "total_time": 0.5095822811126709, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-fp16", - "ttr": 0.5513401031494141, - "ttft": 0.5514194965362549, - "tps": 19.69709805851596, + "ttr": 0.5004770755767822, + "ttft": 0.5005824565887451, + "tps": 19.65654620054879, "num_tokens": 21, - "total_time": 1.5667974948883057, + "total_time": 1.5180552005767822, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { "model": "cloudflare/llama-2-7b-chat-int8", - "ttr": 0.29263997077941895, - "ttft": 0.29274797439575195, - "tps": 34.37024411429695, + "ttr": 0.4272899627685547, + "ttft": 0.42737770080566406, + "tps": 35.81934659623446, "num_tokens": 21, - "total_time": 0.8746466636657715, + "total_time": 0.9857351779937744, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ]