diff --git a/website/public/data/latest.json b/website/public/data/latest.json index 672b0e0..da2024b 100644 --- a/website/public/data/latest.json +++ b/website/public/data/latest.json @@ -1,1156 +1,1318 @@ [ { - "time": "2024-04-18T08:07:28.272485", + "time": "2024-04-22T08:06:56.775212", "region": "cdg", "cmd": "--max-tokens=20", "results": [ { - "model": "gpt-4-turbo", - "ttr": 0.58567214012146, - "ttft": 0.585782527923584, - "tps": 22.841728606527962, + "model": "openai/gpt-4-turbo", + "ttr": 0.5983378887176514, + "ttft": 0.5985531806945801, + "tps": 22.638461402979495, "num_tokens": 20, - "total_time": 1.4175934791564941, - "output": "Golden blaze dips low, kissing sea,\nHorizon melts in fiery hues,\nCrimson, orange" + "total_time": 1.4378328323364258, + "output": "Sunset paints the sky with grace,\nHorizon blushes in embrace,\nColors blend in soft ballet" }, { - "model": "gpt-4-0125-preview", - "ttr": 0.4495091438293457, - "ttft": 0.482158899307251, - "tps": 18.46261143868703, + "model": "openai/gpt-4-0125-preview", + "ttr": 0.4481072425842285, + "ttft": 0.5165748596191406, + "tps": 25.597500244116784, "num_tokens": 20, - "total_time": 1.511265754699707, - "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, in splendid grace it falls" + "total_time": 1.2588348388671875, + "output": "Beneath the vast, expansive sky at dusk,\nThe sun dips low, in shades of gold" }, { - "model": "fixie-scentralus.azure/gpt-4-0125-preview", - "ttr": 4.515986204147339, - "ttft": 4.516130447387695, - "tps": 2.7777599971947655, + "model": "azure.scentralus/gpt-4-0125-preview", + "ttr": 0.7193028926849365, + "ttft": 0.7193951606750488, + "tps": 6.5974639115838665, "num_tokens": 20, - "total_time": 11.356174230575562, + "total_time": 3.599289655685425, "output": "Amidst the tranquil evening sky,\nA canvas painted with goodbye,\nThe sun dips low, in" }, { - "model": "gpt-4-1106-preview", - "ttr": 0.5733511447906494, - "ttft": 0.6679275035858154, - "tps": 13.342656185880205, + "model": "openai/gpt-4-1106-preview", + "ttr": 0.5919198989868164, + "ttft": 0.6541166305541992, + "tps": 19.11289186071332, "num_tokens": 20, - "total_time": 2.0919318199157715, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 1.648210048675537, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { - "model": "fixie-westus.azure/gpt-4-1106-preview", - "ttr": 0.7849717140197754, - "ttft": 0.7850854396820068, - "tps": 11.711416782384564, + "model": "azure.westus/gpt-4-1106-preview", + "ttr": 0.7622711658477783, + "ttft": 0.7623574733734131, + "tps": 17.539960571734067, "num_tokens": 20, - "total_time": 2.4074339866638184, - "output": "The sun dips low, horizon kissed,\nPainting skies with amethyst,\nCrimson streaks" + "total_time": 1.8455982208251953, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with hues of gold and pink" }, { - "model": "fixie-eastus2.azure/gpt-4-1106-preview", - "ttr": 0.68135666847229, - "ttft": 0.6814935207366943, - "tps": 13.726684362194499, + "model": "azure.eastus2/gpt-4-1106-preview", + "ttr": 0.9814193248748779, + "ttft": 0.981525182723999, + "tps": 10.94384937511999, "num_tokens": 20, - "total_time": 2.0656588077545166, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber bliss,\nCloud" + "total_time": 2.7176601886749268, + "output": "Amber hues dance across the evening sky,\nHorizon ablaze with the day's last sigh,\n" }, { - "model": "fixie-frcentral.azure/gpt-4-1106-preview", - "ttr": 0.3766794204711914, - "ttft": 0.37692999839782715, - "tps": 12.340906817312502, + "model": "azure.frcentral/gpt-4-1106-preview", + "ttr": 0.3211376667022705, + "ttft": 0.3212122917175293, + "tps": 6.335727059558323, "num_tokens": 20, - "total_time": 1.91652512550354, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 3.3200788497924805, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "fixie-secentral.azure/gpt-4-1106-preview", - "ttr": 0.3115370273590088, - "ttft": 0.31325244903564453, - "tps": 13.890404034593987, + "model": "azure.secentral/gpt-4-1106-preview", + "ttr": 0.45921754837036133, + "ttft": 0.459369421005249, + "tps": 10.738073646802459, "num_tokens": 20, - "total_time": 1.681103229522705, - "output": "Golden sun dips beneath the horizon,\nPainting the sky with hues of passion,\nCrimson," + "total_time": 2.228774309158325, + "output": "Golden orb descends in the sky,\nPainting hues of red and gold,\nSilhouettes of" }, { - "model": "fixie-uksouth.azure/gpt-4-1106-preview", - "ttr": 0.2946205139160156, - "ttft": 0.29477405548095703, - "tps": 5.149367486446138, + "model": "azure.uksouth/gpt-4-1106-preview", + "ttr": 0.660820722579956, + "ttft": 0.6610517501831055, + "tps": 9.109648158683815, "num_tokens": 20, - "total_time": 3.9845476150512695, + "total_time": 2.7467525005340576, "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { - "model": "gpt-3.5-turbo-0125", - "ttr": 0.49135255813598633, - "ttft": 0.5516366958618164, - "tps": 42.99893543720725, + "model": "openai/gpt-3.5-turbo-0125", + "ttr": 0.3197042942047119, + "ttft": 0.33257627487182617, + "tps": 67.47267667261876, "num_tokens": 20, - "total_time": 0.9935081005096436, - "output": "The sun dips below the horizon\nColors streak across the sky\nA fiery display of light\nGolden" + "total_time": 0.6141717433929443, + "output": "The sun dips low, painting the sky in hues\nOf orange, pink, and purple, so" }, { - "model": "gpt-3.5-turbo-1106", - "ttr": 0.2938990592956543, - "ttft": 0.30849623680114746, - "tps": 80.64957029684534, + "model": "openai/gpt-3.5-turbo-1106", + "ttr": 0.28953075408935547, + "ttft": 0.3369176387786865, + "tps": 81.17566719634927, "num_tokens": 20, - "total_time": 0.5440833568572998, - "output": "The sun dips low, painting the sky in gold\nA fiery ball sinking into the horizon\nColors" + "total_time": 0.5709779262542725, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { - "model": "fixie-westus.azure/gpt-3.5-turbo-1106", - "ttr": 0.2456037998199463, - "ttft": 0.2505524158477783, - "tps": 128.90958413067636, + "model": "azure.westus/gpt-3.5-turbo-1106", + "ttr": 0.18707895278930664, + "ttft": 0.18723368644714355, + "tps": 129.634295957823, "num_tokens": 20, - "total_time": 0.39794254302978516, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" + "total_time": 0.3337998390197754, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" }, { - "model": "fixie-eastus2.azure/gpt-3.5-turbo", - "ttr": 0.18584108352661133, - "ttft": 0.18630456924438477, - "tps": 42.98100873947477, + "model": "azure.eastus2/gpt-3.5-turbo", + "ttr": 0.15187859535217285, + "ttft": 0.1524338722229004, + "tps": 47.618031708397, "num_tokens": 20, - "total_time": 0.6283602714538574, + "total_time": 0.5514423847198486, "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" }, { - "model": "claude-3-opus-20240229", - "ttr": 1.0044584274291992, - "ttft": 1.0046007633209229, - "tps": 19.65379510296248, + "model": "anthropic/claude-3-opus-20240229", + "ttr": 0.9283649921417236, + "ttft": 0.9285275936126709, + "tps": 37.925149824560236, "num_tokens": 20, - "total_time": 1.9713351726531982, + "total_time": 1.4295144081115723, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { - "model": "claude-3-sonnet-20240229", - "ttr": 0.6137163639068604, - "ttft": 0.615250825881958, - "tps": 85.205874145314, + "model": "anthropic/claude-3-sonnet-20240229", + "ttr": 0.5854909420013428, + "ttft": 0.5857160091400146, + "tps": 73.09636415835573, "num_tokens": 20, - "total_time": 0.8382401466369629, + "total_time": 0.845646858215332, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { - "model": "claude-3-haiku-20240307", - "ttr": 0.33461952209472656, - "ttft": 0.3349947929382324, - "tps": 84.83353665676663, + "model": "anthropic/claude-3-haiku-20240307", + "ttr": 0.35446763038635254, + "ttft": 0.3547849655151367, + "tps": 94.45667701016728, "num_tokens": 20, - "total_time": 0.5589628219604492, + "total_time": 0.5559353828430176, "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," }, { - "model": "claude-2.1", - "ttr": 0.518427848815918, - "ttft": 0.5194995403289795, - "tps": 15.510706411390277, + "model": "anthropic/claude-2.1", + "ttr": 0.44562673568725586, + "ttft": 0.4463803768157959, + "tps": 31.241005915234442, "num_tokens": 20, - "total_time": 1.744459867477417, - "output": "Here is a nonet poem about a sunset:\n\nGolden sun\nSinking slowly in the" + "total_time": 1.0545554161071777, + "output": "Here is a nonet poem about a sunset:\n\nSun descending\nGolden horizon glowing\nCl" }, { - "model": "claude-instant-1.2", - "ttr": 0.45298027992248535, - "ttft": 0.4555041790008545, - "tps": 148.49334692333636, + "model": "anthropic/claude-instant-1.2", + "ttr": 0.5597796440124512, + "ttft": 0.5632693767547607, + "tps": 152.40289002528198, "num_tokens": 20, - "total_time": 0.5834560394287109, + "total_time": 0.687938928604126, "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" }, { - "model": "command-r-plus", - "ttr": 0.12260007858276367, - "ttft": 0.19914531707763672, - "tps": 31.36471977151179, + "model": "cohere/command-r-plus", + "ttr": 0.11967277526855469, + "ttft": 0.22998404502868652, + "tps": 29.418396650515717, "num_tokens": 17, - "total_time": 0.7092726230621338, + "total_time": 0.7738614082336426, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { - "model": "command-r", - "ttr": 0.13317465782165527, - "ttft": 0.20613622665405273, - "tps": 86.31790938353835, + "model": "cohere/command-r", + "ttr": 0.13445353507995605, + "ttft": 0.18753862380981445, + "tps": 92.68372855402909, "num_tokens": 18, - "total_time": 0.40308260917663574, - "output": "Orange and pink skies, so serene,\nThe sun melts into the horizon's kiss.\nSoft" + "total_time": 0.3709580898284912, + "output": "Orange and pink skies, so serene\nThe sun melts into the sea\nSoftly the stars begin" }, { - "model": "command-light", - "ttr": 0.13529753684997559, - "ttft": 0.19019627571105957, - "tps": 80.91993594785038, + "model": "cohere/command-light", + "ttr": 0.13283348083496094, + "ttft": 0.18692946434020996, + "tps": 80.46472010129303, "num_tokens": 18, - "total_time": 0.4002804756164551, - "output": "Here is a short story about a sunset: \n\nAs the sun began its descent into the horizon" + "total_time": 0.39820218086242676, + "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { - "model": "fixie-mistral.eastus2.azure", - "ttr": 0.796921968460083, - "ttft": 0.7970733642578125, + "model": "azure.eastus2/mistral", + "ttr": 0.9695878028869629, + "ttft": 0.9696927070617676, "tps": 999, "num_tokens": 20, - "total_time": 0.7974996566772461, + "total_time": 0.9700102806091309, "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" }, { - "model": "api.fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.19990324974060059, - "ttft": 0.20740461349487305, - "tps": 181.9633889326529, + "model": "fireworks.ai/mixtral-8x7b-instruct", + "ttr": 0.20833659172058105, + "ttft": 0.20839643478393555, + "tps": 170.47280817156, "num_tokens": 20, - "total_time": 0.31182122230529785, + "total_time": 0.3198511600494385, "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" }, { - "model": "api.groq.com/mixtral-8x7b-32768", - "ttr": 0.1646876335144043, - "ttft": 0.2608060836791992, - "tps": 642.7481812462697, + "model": "groq.com/mixtral-8x7b-32768", + "ttr": 0.17887043952941895, + "ttft": 0.2371835708618164, + "tps": 802.6971796937953, "num_tokens": 20, - "total_time": 0.29036664962768555, + "total_time": 0.26085376739501953, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { - "model": "text.octoai.run/mixtral-8x7b-instruct", - "ttr": 0.1541743278503418, - "ttft": 0.18517684936523438, - "tps": 68.04417778013058, + "model": "octoai.run/mixtral-8x7b-instruct", + "ttr": 0.14362049102783203, + "ttft": 0.17533636093139648, + "tps": 46.97170738262787, "num_tokens": 20, - "total_time": 0.46440720558166504, - "output": " Radiant sun descends, a fiery sphere,\nIts brilliant hues of red and" + "total_time": 0.5798351764678955, + "output": " Radiant sun, your day is done,\nDescending low, the race is won.\n" + }, + { + "model": "perplexity.ai/mixtral-8x7b-instruct", + "ttr": 0.12900900840759277, + "ttft": 0.1293046474456787, + "tps": 153.15151072555568, + "num_tokens": 20, + "total_time": 0.25336480140686035, + "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." }, { - "model": "api.perplexity.ai/sonar-medium-chat", - "ttr": 0.28775572776794434, - "ttft": 0.2929506301879883, - "tps": 89.55923319495359, + "model": "perplexity.ai/sonar-medium-chat", + "ttr": 0.1454944610595703, + "ttft": 0.14555978775024414, + "tps": 65.52770126151786, "num_tokens": 20, - "total_time": 0.5051007270812988, + "total_time": 0.4355134963989258, "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" }, { - "model": "fixie-llama-2-70b.westus3.azure", - "ttr": 1.2796719074249268, - "ttft": 1.2797832489013672, + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.3684380054473877, + "ttft": 0.38796567916870117, + "tps": 98.62086820070738, + "num_tokens": 20, + "total_time": 0.580622673034668, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "groq.com/llama3-70b-8192", + "ttr": 0.16277265548706055, + "ttft": 3.157120943069458, + "tps": 407.86422912359, + "num_tokens": 20, + "total_time": 3.203705072402954, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" + }, + { + "model": "perplexity.ai/llama-3-70b-instruct", + "ttr": 0.17601966857910156, + "ttft": 0.17620062828063965, + "tps": 41.48575777625885, + "num_tokens": 20, + "total_time": 0.6341891288757324, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "together.xyz/Llama-3-70b-chat-hf", + "ttr": 0.29582691192626953, + "ttft": 0.2958822250366211, + "tps": 102.94472447495356, + "num_tokens": 20, + "total_time": 0.48044729232788086, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "azure.westus3/llama-2-70b", + "ttr": 7.5272979736328125, + "ttft": 7.527413606643677, "tps": 999, "num_tokens": 20, - "total_time": 1.2800929546356201, + "total_time": 7.527679443359375, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "fixie-llama-2-70b.eastus2.azure", - "ttr": 1.3164820671081543, - "ttft": 1.316610336303711, + "model": "azure.eastus2/llama-2-70b", + "ttr": 1.4821009635925293, + "ttft": 1.4822006225585938, "tps": 999, "num_tokens": 20, - "total_time": 1.3169324398040771, + "total_time": 1.482433557510376, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.fireworks.ai/llama-v2-70b-chat", - "ttr": 0.16662049293518066, - "ttft": 0.17497539520263672, - "tps": 147.40211822240965, + "model": "fireworks.ai/llama-v2-70b-chat", + "ttr": 0.1928112506866455, + "ttft": 0.2099165916442871, + "tps": 112.6621201325507, "num_tokens": 20, - "total_time": 0.30387449264526367, + "total_time": 0.37856245040893555, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { - "model": "api.groq.com/llama2-70b-4096", - "ttr": 0.15419983863830566, - "ttft": 0.2661886215209961, - "tps": 151.17005939680527, + "model": "groq.com/llama2-70b-4096", + "ttr": 0.15791654586791992, + "ttft": 0.3874807357788086, + "tps": 364.5493222777032, "num_tokens": 21, - "total_time": 0.39848995208740234, + "total_time": 0.44234299659729004, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { - "model": "text.octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.09619569778442383, - "ttft": 0.15708208084106445, - "tps": 25.00182780025839, + "model": "octoai.run/llama-2-70b-chat-fp16", + "ttr": 0.09893131256103516, + "ttft": 0.1474466323852539, + "tps": 28.342643506997657, "num_tokens": 20, - "total_time": 0.9170265197753906, + "total_time": 0.8178145885467529, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.perplexity.ai/pplx-70b-chat", - "ttr": 0.1495959758758545, - "ttft": 0.1538410186767578, - "tps": 54.99253763271683, + "model": "together.xyz/llama-2-70b-chat", + "ttr": 0.29886674880981445, + "ttft": 0.2990248203277588, + "tps": 78.44975940903613, "num_tokens": 20, - "total_time": 0.4993424415588379, - "output": "Golden orb descends,\nA fiery ball in twilight's dance,\nC" + "total_time": 0.5412180423736572, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "togethercomputer/llama-2-70b-chat", - "ttr": 0.3437161445617676, - "ttft": 0.34638261795043945, - "tps": 66.58231701967848, + "model": "fireworks.ai/llama-v2-13b-chat", + "ttr": 0.23177242279052734, + "ttft": 0.2420203685760498, + "tps": 182.20468933981138, "num_tokens": 20, - "total_time": 0.6317436695098877, - "output": "\n\nA sunset's colors dance and play\nAcross the sky, a fiery" + "total_time": 0.3462986946105957, + "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, { - "model": "api.fireworks.ai/llama-v2-13b-chat", - "ttr": 0.23691987991333008, - "ttft": 0.24515080451965332, - "tps": 206.21337135434322, + "model": "together.xyz/llama-2-13b-chat", + "ttr": 0.2873845100402832, + "ttft": 0.2879202365875244, + "tps": 43.789288595147404, "num_tokens": 20, - "total_time": 0.33728837966918945, - "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.7218163013458252, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "text.octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.09471607208251953, - "ttft": 0.11553478240966797, - "tps": 49.75729172520152, + "model": "octoai.run/llama-2-13b-chat-fp16", + "ttr": 0.10034346580505371, + "ttft": 0.12205886840820312, + "tps": 48.04069352119163, "num_tokens": 20, - "total_time": 0.4973883628845215, + "total_time": 0.51755690574646, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "togethercomputer/llama-2-13b-chat", - "ttr": 0.3252074718475342, - "ttft": 0.32579851150512695, - "tps": 50.11834432654622, + "model": "fireworks.ai/llama-v3-8b-instruct", + "ttr": 0.22245573997497559, + "ttft": 0.2225174903869629, + "tps": 297.2553302597616, "num_tokens": 20, - "total_time": 0.7049012184143066, - "output": "\n\nHere's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.286435604095459, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" }, { - "model": "api.fireworks.ai/llama-v2-7b-chat", - "ttr": 0.2343144416809082, - "ttft": 0.23824238777160645, - "tps": 123.30213998480612, + "model": "groq.com/llama3-8b-8192", + "ttr": 0.16103720664978027, + "ttft": 4.76821494102478, + "tps": 999, "num_tokens": 20, - "total_time": 0.3923354148864746, - "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + "total_time": 4.783697605133057, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "api.perplexity.ai/pplx-7b-chat", - "ttr": 0.16188430786132812, - "ttft": 0.16209149360656738, - "tps": 77.38966561689425, + "model": "perplexity.ai/llama-3-8b-instruct", + "ttr": 0.12770795822143555, + "ttft": 0.12789368629455566, + "tps": 140.02631430981165, "num_tokens": 20, - "total_time": 0.40760231018066406, - "output": "A sunset's fiery hues,\nA canvas painted with the evening's blues" + "total_time": 0.2635824680328369, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "togethercomputer/llama-2-7b-chat", - "ttr": 0.5466136932373047, - "ttft": 0.5467257499694824, - "tps": 97.68615116070968, + "model": "together.xyz/Llama-3-8b-chat-hf", + "ttr": 0.27105069160461426, + "ttft": 0.27110719680786133, + "tps": 205.22244855389513, "num_tokens": 20, - "total_time": 0.7412261962890625, - "output": "\n\n 1. Write a haiku about a sunset.\n\n 2. Write" + "total_time": 0.363689661026001, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "@cf/meta/llama-2-7b-chat-fp16", - "ttr": 0.41291308403015137, - "ttft": 0.41327714920043945, - "tps": 19.833400164935348, - "num_tokens": 21, - "total_time": 1.4216771125793457, + "model": "fireworks.ai/llama-v2-7b-chat", + "ttr": 0.24669766426086426, + "ttft": 0.25202465057373047, + "tps": 179.67861040214285, + "num_tokens": 20, + "total_time": 0.3577690124511719, + "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + }, + { + "model": "together.xyz/llama-2-7b-chat", + "ttr": 0.29149770736694336, + "ttft": 0.2920844554901123, + "tps": 101.12810204713539, + "num_tokens": 20, + "total_time": 0.4799649715423584, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "@cf/meta/llama-2-7b-chat-int8", - "ttr": 0.21407437324523926, - "ttft": 0.21425771713256836, - "tps": 31.290548313312513, + "model": "cloudflare/llama-2-7b-chat-fp16", + "ttr": 0.42386579513549805, + "ttft": 0.42392396926879883, + "tps": 19.483144167604213, "num_tokens": 21, - "total_time": 0.8534283638000488, + "total_time": 1.4504523277282715, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "Neets-7B", - "ttr": 0.5406002998352051, - "ttft": 0.5542805194854736, - "tps": 75.782533984414, - "num_tokens": 20, - "total_time": 0.8049979209899902, - "output": " A golden light descends, as day surrenders,\nBathing the sky in h" + "model": "cloudflare/llama-2-7b-chat-int8", + "ttr": 0.3685901165008545, + "ttft": 0.36864519119262695, + "tps": 36.242228121982464, + "num_tokens": 21, + "total_time": 0.920487642288208, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ] }, { - "time": "2024-04-18T08:06:29.969357", + "time": "2024-04-22T08:06:20.506448", "region": "sea", "cmd": "--max-tokens=20", "results": [ { - "model": "gpt-4-turbo", - "ttr": 0.5251643657684326, - "ttft": 0.5253849029541016, - "tps": 26.514846352734423, + "model": "openai/gpt-4-turbo", + "ttr": 0.5042407512664795, + "ttft": 0.504305362701416, + "tps": 23.47526167896962, "num_tokens": 20, - "total_time": 1.24196457862854, - "output": "Golden blaze dips low, kissing sea,\nHorizon melts in fiery hues,\nCrimson, orange" + "total_time": 1.3136680126190186, + "output": "Sunset paints the sky with grace,\nHorizon blushes in embrace,\nColors blend in soft ballet" }, { - "model": "gpt-4-0125-preview", - "ttr": 0.39721107482910156, - "ttft": 0.4652717113494873, - "tps": 35.23959694476098, + "model": "openai/gpt-4-0125-preview", + "ttr": 0.30695271492004395, + "ttft": 0.3646430969238281, + "tps": 24.450580185929493, "num_tokens": 20, - "total_time": 1.0044379234313965, - "output": "Beneath the vast, expansive sky at dusk,\nThe sun dips low, in shades of gold" + "total_time": 1.1417207717895508, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { - "model": "fixie-scentralus.azure/gpt-4-0125-preview", - "ttr": 0.5662639141082764, - "ttft": 0.5663607120513916, - "tps": 5.879744528617168, + "model": "azure.scentralus/gpt-4-0125-preview", + "ttr": 0.49054574966430664, + "ttft": 0.49062681198120117, + "tps": 11.871399071819992, "num_tokens": 20, - "total_time": 3.7977936267852783, - "output": "Amidst the tranquil evening sky,\nA canvas painted with goodbye,\nThe sun dips low, in" + "total_time": 2.0911121368408203, + "output": "Beneath the vast, ever-changing sky,\nColors blend where the sun bids goodbye,\nOranges" }, { - "model": "gpt-4-1106-preview", - "ttr": 0.40828514099121094, - "ttft": 0.5122613906860352, - "tps": 28.62767536530097, + "model": "openai/gpt-4-1106-preview", + "ttr": 0.3850069046020508, + "ttft": 0.4440650939941406, + "tps": 26.919623289059437, "num_tokens": 20, - "total_time": 1.175954818725586, - "output": "Amber hues dance across the evening sky,\nSoft whispers of light bid the day goodbye,\nCrim" + "total_time": 1.1498699188232422, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { - "model": "fixie-westus.azure/gpt-4-1106-preview", - "ttr": 0.27032041549682617, - "ttft": 0.27045631408691406, - "tps": 20.20026843528086, + "model": "azure.westus/gpt-4-1106-preview", + "ttr": 0.3797454833984375, + "ttft": 0.3798031806945801, + "tps": 10.190510083188462, "num_tokens": 20, - "total_time": 1.2110378742218018, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber bliss,\nCloud" + "total_time": 2.2442829608917236, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "fixie-eastus2.azure/gpt-4-1106-preview", - "ttr": 0.48454737663269043, - "ttft": 0.4847226142883301, - "tps": 2.352739433729584, + "model": "azure.eastus2/gpt-4-1106-preview", + "ttr": 0.4943256378173828, + "ttft": 0.4943814277648926, + "tps": 6.786844766628102, "num_tokens": 20, - "total_time": 8.560415029525757, + "total_time": 3.293915033340454, "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "fixie-frcentral.azure/gpt-4-1106-preview", - "ttr": 0.8742132186889648, - "ttft": 0.8743431568145752, - "tps": 6.436975866442149, + "model": "azure.frcentral/gpt-4-1106-preview", + "ttr": 1.2634272575378418, + "ttft": 1.263505458831787, + "tps": 8.617590999581513, "num_tokens": 20, - "total_time": 3.8260397911071777, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 3.4682979583740234, + "output": "The sun dips low, horizon kissed,\nPainting skies with amethyst,\nCrimson streaks" }, { - "model": "fixie-secentral.azure/gpt-4-1106-preview", - "ttr": 0.8657791614532471, - "ttft": 0.8659193515777588, - "tps": 13.384530555065224, + "model": "azure.secentral/gpt-4-1106-preview", + "ttr": 0.9889154434204102, + "ttft": 0.9890758991241455, + "tps": 3.9578369882291615, "num_tokens": 20, - "total_time": 2.285468578338623, + "total_time": 5.789677858352661, "output": "Golden sun dips beneath the horizon,\nPainting the sky with hues of passion,\nCrimson," }, { - "model": "fixie-uksouth.azure/gpt-4-1106-preview", - "ttr": 1.5683879852294922, - "ttft": 1.5685138702392578, - "tps": 6.0512924639358525, + "model": "azure.uksouth/gpt-4-1106-preview", + "ttr": 0.8197438716888428, + "ttft": 0.8198344707489014, + "tps": 4.412255976234034, "num_tokens": 20, - "total_time": 4.708338975906372, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 5.12602162361145, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "gpt-3.5-turbo-0125", - "ttr": 0.32707858085632324, - "ttft": 0.3504014015197754, - "tps": 63.08352423939699, + "model": "openai/gpt-3.5-turbo-0125", + "ttr": 0.2534823417663574, + "ttft": 0.27307987213134766, + "tps": 33.926350510267504, "num_tokens": 20, - "total_time": 0.6515893936157227, + "total_time": 0.8331165313720703, "output": "The sun dips below the horizon line\nPainting the sky in hues divine\nOrange, pink," }, { - "model": "gpt-3.5-turbo-1106", - "ttr": 0.23726415634155273, - "ttft": 0.23756742477416992, - "tps": 80.4938976904857, + "model": "openai/gpt-3.5-turbo-1106", + "ttr": 0.381000280380249, + "ttft": 0.5620617866516113, + "tps": 332.3620409218681, "num_tokens": 20, - "total_time": 0.47361016273498535, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" + "total_time": 0.6192283630371094, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" }, { - "model": "fixie-westus.azure/gpt-3.5-turbo-1106", - "ttr": 0.07235121726989746, - "ttft": 0.07252216339111328, - "tps": 108.5219285941896, + "model": "azure.westus/gpt-3.5-turbo-1106", + "ttr": 0.0831611156463623, + "ttft": 0.08329010009765625, + "tps": 72.19085717598632, "num_tokens": 20, - "total_time": 0.24760198593139648, - "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nCasting a" + "total_time": 0.3464813232421875, + "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { - "model": "fixie-eastus2.azure/gpt-3.5-turbo", - "ttr": 0.15089035034179688, - "ttft": 0.15096044540405273, - "tps": 52.86609301810303, + "model": "azure.eastus2/gpt-3.5-turbo", + "ttr": 0.23638224601745605, + "ttft": 0.23682498931884766, + "tps": 54.292460131459904, "num_tokens": 20, - "total_time": 0.5103590488433838, - "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" + "total_time": 0.5867815017700195, + "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, bidding farewell.\nCrim" }, { - "model": "claude-3-opus-20240229", - "ttr": 1.1011734008789062, - "ttft": 1.1013455390930176, - "tps": 22.718520412715815, + "model": "anthropic/claude-3-opus-20240229", + "ttr": 0.8946642875671387, + "ttft": 0.8947696685791016, + "tps": 39.140959322599976, "num_tokens": 20, - "total_time": 1.9376676082611084, + "total_time": 1.3801946640014648, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { - "model": "claude-3-sonnet-20240229", - "ttr": 0.5130131244659424, - "ttft": 0.5132133960723877, - "tps": 73.89315819682088, + "model": "anthropic/claude-3-sonnet-20240229", + "ttr": 0.6642239093780518, + "ttft": 0.6666467189788818, + "tps": 71.55118323033508, "num_tokens": 20, - "total_time": 0.7703413963317871, + "total_time": 0.9321908950805664, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { - "model": "claude-3-haiku-20240307", - "ttr": 0.28017497062683105, - "ttft": 0.2810368537902832, - "tps": 109.82334944793257, + "model": "anthropic/claude-3-haiku-20240307", + "ttr": 0.281796932220459, + "ttft": 0.28218507766723633, + "tps": 99.60625346845518, "num_tokens": 20, - "total_time": 0.4540419578552246, - "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," + "total_time": 0.47293615341186523, + "output": "Here is a nonet about a sunset:\n\nSunset's golden glow\nPaints" }, { - "model": "claude-2.1", - "ttr": 0.432767391204834, - "ttft": 0.43323445320129395, - "tps": 15.343228201799867, + "model": "anthropic/claude-2.1", + "ttr": 0.41651487350463867, + "ttft": 0.41733622550964355, + "tps": 32.08855906696007, "num_tokens": 20, - "total_time": 1.6715657711029053, - "output": "Here is a nonet poem about a sunset:\n\nGolden sun\nSinking slowly in the" + "total_time": 1.0094475746154785, + "output": "Here is a nonet poem about a sunset:\n\nSun descending\nGolden horizon glowing\nCl" }, { - "model": "claude-instant-1.2", - "ttr": 0.4096846580505371, - "ttft": 0.41107797622680664, - "tps": 147.81386514719847, + "model": "anthropic/claude-instant-1.2", + "ttr": 0.4013829231262207, + "ttft": 0.40471959114074707, + "tps": 170.99551546413872, "num_tokens": 20, - "total_time": 0.5396180152893066, + "total_time": 0.5158336162567139, "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" }, { - "model": "command-r-plus", - "ttr": 0.06811404228210449, - "ttft": 0.1499326229095459, - "tps": 31.307376368525453, + "model": "cohere/command-r-plus", + "ttr": 0.06077742576599121, + "ttft": 0.5973734855651855, + "tps": 31.709284584660875, "num_tokens": 17, - "total_time": 0.660994291305542, + "total_time": 1.1019575595855713, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { - "model": "command-r", - "ttr": 0.08307480812072754, - "ttft": 0.13100242614746094, - "tps": 88.62849213821343, - "num_tokens": 19, - "total_time": 0.33409738540649414, - "output": "Vibrant hues of heaven kiss the sky,\nOrange and pink blend, birds fly high." + "model": "cohere/command-r", + "ttr": 0.07931971549987793, + "ttft": 0.1299593448638916, + "tps": 92.57626880980511, + "num_tokens": 18, + "total_time": 0.31359171867370605, + "output": "The horizon's fiery blaze,\nThe sky a canvas of amber and red,\nThe sun slowly" }, { - "model": "command-light", - "ttr": 0.08389449119567871, - "ttft": 0.14298677444458008, - "tps": 75.18639492509111, + "model": "cohere/command-light", + "ttr": 0.08011507987976074, + "ttft": 0.1263134479522705, + "tps": 82.85998098837223, "num_tokens": 18, - "total_time": 0.3690915107727051, - "output": "Here is a short story about a sunset: \n\nAs the sun began its descent into the horizon" + "total_time": 0.3314788341522217, + "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { - "model": "fixie-mistral.eastus2.azure", - "ttr": 0.830927848815918, - "ttft": 0.8310329914093018, + "model": "azure.eastus2/mistral", + "ttr": 0.9781889915466309, + "ttft": 0.9782519340515137, "tps": 999, "num_tokens": 20, - "total_time": 0.8312356472015381, - "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink," + "total_time": 0.9783940315246582, + "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" }, { - "model": "api.fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.12719035148620605, - "ttft": 0.12896966934204102, - "tps": 247.2726640871532, + "model": "fireworks.ai/mixtral-8x7b-instruct", + "ttr": 0.13983511924743652, + "ttft": 0.143540620803833, + "tps": 199.86902086677367, "num_tokens": 20, - "total_time": 0.20580792427062988, + "total_time": 0.238602876663208, "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" }, { - "model": "api.groq.com/mixtral-8x7b-32768", - "ttr": 0.033103227615356445, - "ttft": 0.11828851699829102, - "tps": 871.0435676030168, + "model": "groq.com/mixtral-8x7b-32768", + "ttr": 0.034388065338134766, + "ttft": 0.11794114112854004, + "tps": 729.4775596137123, "num_tokens": 20, - "total_time": 0.14010143280029297, + "total_time": 0.14398717880249023, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { - "model": "text.octoai.run/mixtral-8x7b-instruct", - "ttr": 0.13048744201660156, - "ttft": 0.15773391723632812, - "tps": 58.28323602612119, + "model": "octoai.run/mixtral-8x7b-instruct", + "ttr": 0.25011777877807617, + "ttft": 0.27352166175842285, + "tps": 56.69756730240424, "num_tokens": 20, - "total_time": 0.48372817039489746, + "total_time": 0.6086330413818359, "output": " Radiant sun descends, a fiery sphere,\nIts brilliant hues of red and" }, { - "model": "api.perplexity.ai/sonar-medium-chat", - "ttr": 0.1785731315612793, - "ttft": 0.1786954402923584, - "tps": 84.97764015026706, + "model": "perplexity.ai/mixtral-8x7b-instruct", + "ttr": 0.13250970840454102, + "ttft": 0.13263940811157227, + "tps": 152.36559279582434, "num_tokens": 20, - "total_time": 0.4022836685180664, - "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" + "total_time": 0.2573394775390625, + "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." + }, + { + "model": "perplexity.ai/sonar-medium-chat", + "ttr": 0.142625093460083, + "ttft": 0.14266490936279297, + "tps": 75.10567337252677, + "num_tokens": 20, + "total_time": 0.3956418037414551, + "output": "Radiant sun, descending low,\nCasting hues of crimson glow.\nS" + }, + { + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.21239638328552246, + "ttft": 0.2136092185974121, + "tps": 66.53862101384597, + "num_tokens": 20, + "total_time": 0.4991576671600342, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "groq.com/llama3-70b-8192", + "ttr": 0.04072880744934082, + "ttft": 0.1012113094329834, + "tps": 281.57847204064757, + "num_tokens": 20, + "total_time": 0.16868805885314941, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" + }, + { + "model": "perplexity.ai/llama-3-70b-instruct", + "ttr": 0.18464970588684082, + "ttft": 0.18507742881774902, + "tps": 38.33196696472807, + "num_tokens": 20, + "total_time": 0.6807472705841064, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" }, { - "model": "fixie-llama-2-70b.westus3.azure", - "ttr": 7.403578281402588, - "ttft": 7.40370512008667, + "model": "together.xyz/Llama-3-70b-chat-hf", + "ttr": 0.17232275009155273, + "ttft": 0.17242932319641113, + "tps": 100.58866979529293, + "num_tokens": 20, + "total_time": 0.36131739616394043, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "azure.westus3/llama-2-70b", + "ttr": 7.415329933166504, + "ttft": 7.415436267852783, "tps": 999, "num_tokens": 20, - "total_time": 7.403924226760864, + "total_time": 7.41561484336853, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "fixie-llama-2-70b.eastus2.azure", - "ttr": 1.4443376064300537, - "ttft": 1.4444644451141357, + "model": "azure.eastus2/llama-2-70b", + "ttr": 1.938852310180664, + "ttft": 1.9389588832855225, "tps": 999, "num_tokens": 20, - "total_time": 1.4446940422058105, + "total_time": 1.9391627311706543, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.fireworks.ai/llama-v2-70b-chat", - "ttr": 0.154526948928833, - "ttft": 0.16694140434265137, - "tps": 195.67500356032667, + "model": "fireworks.ai/llama-v2-70b-chat", + "ttr": 0.1957550048828125, + "ttft": 0.21876811981201172, + "tps": 188.54729310891446, "num_tokens": 20, - "total_time": 0.2640411853790283, + "total_time": 0.31953859329223633, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { - "model": "api.groq.com/llama2-70b-4096", - "ttr": 0.03147125244140625, - "ttft": 0.21052312850952148, - "tps": 557.3233410401552, + "model": "groq.com/llama2-70b-4096", + "ttr": 0.023345470428466797, + "ttft": 0.2662339210510254, + "tps": 454.4725022889928, "num_tokens": 21, - "total_time": 0.24640893936157227, + "total_time": 0.3102409839630127, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { - "model": "text.octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.07215142250061035, - "ttft": 0.14069819450378418, - "tps": 24.89053468636876, + "model": "octoai.run/llama-2-70b-chat-fp16", + "ttr": 0.2037487030029297, + "ttft": 0.2511575222015381, + "tps": 26.908543044099112, "num_tokens": 20, - "total_time": 0.9040405750274658, + "total_time": 0.9572529792785645, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.perplexity.ai/pplx-70b-chat", - "ttr": 0.15520834922790527, - "ttft": 0.15542364120483398, - "tps": 42.96911996954647, + "model": "together.xyz/llama-2-70b-chat", + "ttr": 0.21956515312194824, + "ttft": 0.21961188316345215, + "tps": 74.50217780092123, "num_tokens": 20, - "total_time": 0.5976016521453857, - "output": "Golden orb descends,\nA fiery ball in twilight's dance,\nC" + "total_time": 0.4746379852294922, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "togethercomputer/llama-2-70b-chat", - "ttr": 0.12778759002685547, - "ttft": 0.12796759605407715, - "tps": 63.671005051061904, + "model": "fireworks.ai/llama-v2-13b-chat", + "ttr": 0.18693780899047852, + "ttft": 0.19986724853515625, + "tps": 197.511093486666, "num_tokens": 20, - "total_time": 0.4263765811920166, - "output": "\n\nA sunset's colors dance and play\nAcross the sky, a fiery" + "total_time": 0.2960643768310547, + "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, { - "model": "api.fireworks.ai/llama-v2-13b-chat", - "ttr": 0.188645601272583, - "ttft": 0.20050573348999023, - "tps": 334.2046868971533, + "model": "together.xyz/llama-2-13b-chat", + "ttr": 0.1624457836151123, + "ttft": 0.1627030372619629, + "tps": 42.29470937829384, "num_tokens": 20, - "total_time": 0.257357120513916, - "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.6119318008422852, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "text.octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.07470369338989258, - "ttft": 0.10086297988891602, - "tps": 54.44100397658454, + "model": "octoai.run/llama-2-13b-chat-fp16", + "ttr": 0.0747978687286377, + "ttft": 0.09755325317382812, + "tps": 52.313744179741526, "num_tokens": 20, - "total_time": 0.44986462593078613, + "total_time": 0.46074652671813965, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "togethercomputer/llama-2-13b-chat", - "ttr": 0.15811514854431152, - "ttft": 0.15819454193115234, - "tps": 40.34405640853256, + "model": "fireworks.ai/llama-v3-8b-instruct", + "ttr": 0.17707538604736328, + "ttft": 0.18795156478881836, + "tps": 357.7408198811298, "num_tokens": 20, - "total_time": 0.6291437149047852, - "output": "\n\nHere's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.24106264114379883, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" }, { - "model": "api.fireworks.ai/llama-v2-7b-chat", - "ttr": 0.18305635452270508, - "ttft": 0.19239449501037598, - "tps": 242.84575113207663, + "model": "groq.com/llama3-8b-8192", + "ttr": 0.04481387138366699, + "ttft": 0.15939688682556152, + "tps": 789.4104664639281, "num_tokens": 20, - "total_time": 0.2706334590911865, - "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + "total_time": 0.18346548080444336, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "api.perplexity.ai/pplx-7b-chat", - "ttr": 0.13318729400634766, - "ttft": 0.1336820125579834, - "tps": 129.7127891778746, + "model": "perplexity.ai/llama-3-8b-instruct", + "ttr": 0.1204688549041748, + "ttft": 0.12056541442871094, + "tps": 129.21516951309877, "num_tokens": 20, - "total_time": 0.28015947341918945, - "output": "A sunset's fiery hues,\nA canvas painted with the evening's blues" + "total_time": 0.2676069736480713, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "togethercomputer/llama-2-7b-chat", - "ttr": 0.16141748428344727, - "ttft": 0.16165757179260254, - "tps": 72.72228990260413, + "model": "together.xyz/Llama-3-8b-chat-hf", + "ttr": 0.1269392967224121, + "ttft": 0.12736988067626953, + "tps": 217.38369590419947, "num_tokens": 20, - "total_time": 0.4229254722595215, - "output": "\n\n 1. Write a haiku about a sunset.\n\n 2. Write" + "total_time": 0.21477293968200684, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "@cf/meta/llama-2-7b-chat-fp16", - "ttr": 0.327134370803833, - "ttft": 0.3271908760070801, - "tps": 19.680471246025128, - "num_tokens": 21, - "total_time": 1.3434267044067383, + "model": "fireworks.ai/llama-v2-7b-chat", + "ttr": 0.16161346435546875, + "ttft": 0.17016339302062988, + "tps": 280.83326931412523, + "num_tokens": 20, + "total_time": 0.23781919479370117, + "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + }, + { + "model": "together.xyz/llama-2-7b-chat", + "ttr": 0.14069080352783203, + "ttft": 0.14288878440856934, + "tps": 102.78804388483455, + "num_tokens": 20, + "total_time": 0.32773518562316895, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "@cf/meta/llama-2-7b-chat-int8", - "ttr": 0.5843923091888428, - "ttft": 0.5845048427581787, - "tps": 30.423247072681917, + "model": "cloudflare/llama-2-7b-chat-fp16", + "ttr": 0.5737338066101074, + "ttft": 0.5739378929138184, + "tps": 19.684526005250703, "num_tokens": 21, - "total_time": 1.2418968677520752, + "total_time": 1.5899643898010254, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "Neets-7B", - "ttr": 0.5222773551940918, - "ttft": 0.5378925800323486, - "tps": 74.3607081138984, - "num_tokens": 20, - "total_time": 0.7934038639068604, - "output": " A golden light descends, as day surrenders,\nBathing the sky in h" + "model": "cloudflare/llama-2-7b-chat-int8", + "ttr": 1.0605113506317139, + "ttft": 1.0606534481048584, + "tps": 36.519859172772826, + "num_tokens": 21, + "total_time": 1.6083006858825684, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ] }, { - "time": "2024-04-18T08:07:01.351021", + "time": "2024-04-22T08:06:42.176632", "region": "iad", "cmd": "--max-tokens=20", "results": [ { - "model": "gpt-4-turbo", - "ttr": 0.8567218780517578, - "ttft": 0.8572628498077393, - "tps": 20.288528059231304, + "model": "openai/gpt-4-turbo", + "ttr": 0.4815518856048584, + "ttft": 0.4816761016845703, + "tps": 26.0290968049171, "num_tokens": 20, - "total_time": 1.793752670288086, - "output": "Amber hues paint the evening sky,\nHorizon blushes as the sun sighs,\nClouds" + "total_time": 1.2116284370422363, + "output": "Sunset paints the sky with grace,\nHorizon's blush at day's embrace,\nColors meld in" }, { - "model": "gpt-4-0125-preview", - "ttr": 0.7304298877716064, - "ttft": 0.8078732490539551, - "tps": 25.579153502635375, + "model": "openai/gpt-4-0125-preview", + "ttr": 0.4113337993621826, + "ttft": 0.4861607551574707, + "tps": 16.533285643954947, "num_tokens": 20, - "total_time": 1.5506656169891357, - "output": "Amidst the tranquil evening sky,\nA canvas painted with goodbye,\nThe sun dips low, in" + "total_time": 1.6353576183319092, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye,\n" }, { - "model": "fixie-scentralus.azure/gpt-4-0125-preview", - "ttr": 0.4981348514556885, - "ttft": 0.49829912185668945, - "tps": 6.749557528045104, + "model": "azure.scentralus/gpt-4-0125-preview", + "ttr": 0.5049262046813965, + "ttft": 0.5050053596496582, + "tps": 9.412204010555984, "num_tokens": 20, - "total_time": 3.313298463821411, - "output": "Amidst the tranquil evening sky,\nA canvas painted with goodbye,\nThe sun dips low, in" + "total_time": 2.5236611366271973, + "output": "Beneath the vast, expansive evening sky,\nThe sun dips low, bidding the day goodbye.\n" }, { - "model": "gpt-4-1106-preview", - "ttr": 0.44886207580566406, - "ttft": 0.5558428764343262, - "tps": 22.036473158907388, + "model": "openai/gpt-4-1106-preview", + "ttr": 0.4584345817565918, + "ttft": 0.5304193496704102, + "tps": 25.981498090635554, "num_tokens": 20, - "total_time": 1.4180498123168945, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 1.2617089748382568, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "fixie-westus.azure/gpt-4-1106-preview", - "ttr": 0.4561307430267334, - "ttft": 0.45626401901245117, - "tps": 18.47041658680149, + "model": "azure.westus/gpt-4-1106-preview", + "ttr": 0.5812273025512695, + "ttft": 0.5815980434417725, + "tps": 17.205045722215843, "num_tokens": 20, - "total_time": 1.484935998916626, - "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" + "total_time": 1.6859252452850342, + "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { - "model": "fixie-eastus2.azure/gpt-4-1106-preview", - "ttr": 0.22776103019714355, - "ttft": 0.22788715362548828, - "tps": 15.778862458271789, + "model": "azure.eastus2/gpt-4-1106-preview", + "ttr": 0.4476635456085205, + "ttft": 0.44777560234069824, + "tps": 14.149077063612612, "num_tokens": 20, - "total_time": 1.4320297241210938, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 1.7906193733215332, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with shades of gold and pink" }, { - "model": "fixie-frcentral.azure/gpt-4-1106-preview", - "ttr": 0.7267935276031494, - "ttft": 0.7269408702850342, - "tps": 8.66209872246402, + "model": "azure.frcentral/gpt-4-1106-preview", + "ttr": 0.6240036487579346, + "ttft": 0.6242091655731201, + "tps": 11.061335117394808, "num_tokens": 20, - "total_time": 2.9204046726226807, - "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" + "total_time": 2.3419041633605957, + "output": "The sun dips low, horizon's fiery kiss,\nPainting the sky with hues of gold and pink" }, { - "model": "fixie-secentral.azure/gpt-4-1106-preview", - "ttr": 0.23662853240966797, - "ttft": 0.23789691925048828, - "tps": 13.117374595162959, + "model": "azure.secentral/gpt-4-1106-preview", + "ttr": 0.5716967582702637, + "ttft": 0.5718288421630859, + "tps": 14.748447505561774, "num_tokens": 20, - "total_time": 1.6863574981689453, - "output": "Golden sun dips low, kissing the sea,\nCasting hues of pink and ruby,\nDay surrend" + "total_time": 1.8601000308990479, + "output": "Golden sun dips low, painting sky,\nCrimson, orange, pink hues comply,\nClouds" }, { - "model": "fixie-uksouth.azure/gpt-4-1106-preview", - "ttr": 1.2672183513641357, - "ttft": 1.2673900127410889, - "tps": 2.9159033645276935, + "model": "azure.uksouth/gpt-4-1106-preview", + "ttr": 0.6359684467315674, + "ttft": 0.6361269950866699, + "tps": 8.847335844444899, "num_tokens": 20, - "total_time": 7.78338098526001, + "total_time": 2.7836661338806152, "output": "The sun dips low, horizon's fiery kiss,\nPainting skies in shades of amber and pink,\n" }, { - "model": "gpt-3.5-turbo-0125", - "ttr": 0.5825192928314209, - "ttft": 0.6199846267700195, - "tps": 71.33176452896933, + "model": "openai/gpt-3.5-turbo-0125", + "ttr": 0.35339999198913574, + "ttft": 0.38275861740112305, + "tps": 52.58968291153859, "num_tokens": 20, - "total_time": 0.886345624923706, + "total_time": 0.7440462112426758, "output": "The sun dips below the horizon line\nPainting the sky in hues divine\nOrange, pink," }, { - "model": "gpt-3.5-turbo-1106", - "ttr": 0.2533574104309082, - "ttft": 0.27350878715515137, - "tps": 80.07718750345413, + "model": "openai/gpt-3.5-turbo-1106", + "ttr": 0.3066542148590088, + "ttft": 0.3278391361236572, + "tps": 79.97127560825083, "num_tokens": 20, - "total_time": 0.510779857635498, + "total_time": 0.5654244422912598, "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { - "model": "fixie-westus.azure/gpt-3.5-turbo-1106", - "ttr": 0.19302940368652344, - "ttft": 0.21396732330322266, - "tps": 234.66226936237123, + "model": "azure.westus/gpt-3.5-turbo-1106", + "ttr": 0.13196444511413574, + "ttft": 0.1583099365234375, + "tps": 216.86784391523594, "num_tokens": 20, - "total_time": 0.2949347496032715, + "total_time": 0.24592089653015137, "output": "The sky is painted in hues of gold\nAs the sun dips below the horizon\nA fiery glow" }, { - "model": "fixie-eastus2.azure/gpt-3.5-turbo", - "ttr": 0.1272733211517334, - "ttft": 0.12785124778747559, - "tps": 39.70938698224852, + "model": "azure.eastus2/gpt-3.5-turbo", + "ttr": 0.09393525123596191, + "ttft": 0.10218954086303711, + "tps": 67.47456192043784, "num_tokens": 20, - "total_time": 0.6063275337219238, + "total_time": 0.3837771415710449, "output": "Golden hues paint the sky's vast canvas,\nAs the sun descends, a fiery ball.\nSil" }, { - "model": "claude-3-opus-20240229", - "ttr": 0.8431558609008789, - "ttft": 0.8433949947357178, - "tps": 36.52314050760947, + "model": "anthropic/claude-3-opus-20240229", + "ttr": 0.9463369846343994, + "ttft": 0.9466469287872314, + "tps": 40.08392639690804, "num_tokens": 20, - "total_time": 1.3636131286621094, + "total_time": 1.4206523895263672, "output": "Here is a nonet poem about a sunset:\n\nGolden sun descends in the west," }, { - "model": "claude-3-sonnet-20240229", - "ttr": 0.5483181476593018, - "ttft": 0.5487213134765625, - "tps": 74.30849693923697, + "model": "anthropic/claude-3-sonnet-20240229", + "ttr": 0.6284692287445068, + "ttft": 0.6394884586334229, + "tps": 83.13922719486216, "num_tokens": 20, - "total_time": 0.8044121265411377, + "total_time": 0.86802077293396, "output": "Here's a nonet about a sunset:\n\nBrilliant hues paint the evening sky," }, { - "model": "claude-3-haiku-20240307", - "ttr": 0.38084864616394043, - "ttft": 0.3891561031341553, - "tps": 104.3853934438684, + "model": "anthropic/claude-3-haiku-20240307", + "ttr": 0.26909780502319336, + "ttft": 0.2728614807128906, + "tps": 113.63678960764882, "num_tokens": 20, - "total_time": 0.571173906326294, + "total_time": 0.4400608539581299, "output": "Here is a nonet about a sunset:\n\nVibrant hues paint the sky," }, { - "model": "claude-2.1", - "ttr": 0.6025755405426025, - "ttft": 0.602869987487793, - "tps": 14.257554868625526, + "model": "anthropic/claude-2.1", + "ttr": 0.39328551292419434, + "ttft": 0.4193079471588135, + "tps": 29.582680059067528, "num_tokens": 20, - "total_time": 1.9354968070983887, - "output": "Here is a nonet poem about a sunset:\n\nGolden sun\nSinking slowly in the" + "total_time": 1.0615756511688232, + "output": "Here is a nonet poem about a sunset:\n\nRadiant, glowing\nColors fill the" }, { - "model": "claude-instant-1.2", - "ttr": 0.40815305709838867, - "ttft": 0.41573262214660645, - "tps": 174.8592994374084, + "model": "anthropic/claude-instant-1.2", + "ttr": 0.4070167541503906, + "ttft": 0.4090762138366699, + "tps": 171.46386730726613, "num_tokens": 20, - "total_time": 0.5243914127349854, + "total_time": 0.5198867321014404, "output": "Here is a 9 line (nonet) poem about a sunset:\n\nThe Setting Sun" }, { - "model": "command-r-plus", - "ttr": 0.053931236267089844, - "ttft": 0.13417720794677734, - "tps": 31.51924980003654, + "model": "cohere/command-r-plus", + "ttr": 0.0662691593170166, + "ttft": 0.21483230590820312, + "tps": 33.492437498190846, "num_tokens": 17, - "total_time": 0.641803503036499, + "total_time": 0.6925520896911621, "output": "Here's a nonet about a sunset: \n\n\"Sky's canvas, painted\nOrange and" }, { - "model": "command-r", - "ttr": 0.05968332290649414, - "ttft": 0.10723090171813965, - "tps": 70.90377589336467, - "num_tokens": 17, - "total_time": 0.3328888416290283, - "output": "Orange and pink skies,\nThe sun melts into the sea,\nPeacefully surrenders," + "model": "cohere/command-r", + "ttr": 0.05870628356933594, + "ttft": 0.12244176864624023, + "tps": 90.47431303482789, + "num_tokens": 18, + "total_time": 0.31034040451049805, + "output": "The horizon's fiery blaze,\nThe sky a canvas of amber and red,\nThe sun slowly" }, { - "model": "command-light", - "ttr": 0.05976152420043945, - "ttft": 0.11610984802246094, - "tps": 65.96544230500584, + "model": "cohere/command-light", + "ttr": 0.05734443664550781, + "ttft": 0.13303875923156738, + "tps": 80.71465862499123, "num_tokens": 18, - "total_time": 0.37382054328918457, - "output": "Here is a short story about a sunset: \n\nAs the sun began its descent into the horizon" + "total_time": 0.3436572551727295, + "output": "Here is a non-technical sentence about a sunset: \n\n\"As day turns into night," }, { - "model": "fixie-mistral.eastus2.azure", - "ttr": 0.7454423904418945, - "ttft": 0.745568037033081, + "model": "azure.eastus2/mistral", + "ttr": 0.7112693786621094, + "ttft": 0.7113533020019531, "tps": 999, "num_tokens": 20, - "total_time": 0.7458314895629883, + "total_time": 0.7115473747253418, "output": "Golden orb descends, painting the sky,\nVibrant hues of red and pink kiss" }, { - "model": "api.fireworks.ai/mixtral-8x7b-instruct", - "ttr": 0.13196301460266113, - "ttft": 0.13277935981750488, - "tps": 191.734536950601, + "model": "fireworks.ai/mixtral-8x7b-instruct", + "ttr": 0.14188146591186523, + "ttft": 0.1485135555267334, + "tps": 147.56506587414012, "num_tokens": 20, - "total_time": 0.23187470436096191, + "total_time": 0.2772703170776367, "output": "Radiant orb descends, a fiery sight,\nA blazing path of red and" }, { - "model": "api.groq.com/mixtral-8x7b-32768", - "ttr": 0.1577317714691162, - "ttft": 0.18820691108703613, - "tps": 878.6691364558525, + "model": "groq.com/mixtral-8x7b-32768", + "ttr": 0.20771360397338867, + "ttft": 0.2905716896057129, + "tps": 999, "num_tokens": 20, - "total_time": 0.20983052253723145, + "total_time": 0.2909269332885742, "output": "Radiant sun, your day is done,\nSinking low, the day is won.\n" }, { - "model": "text.octoai.run/mixtral-8x7b-instruct", - "ttr": 0.17316150665283203, - "ttft": 0.17727184295654297, - "tps": 68.77581603116548, + "model": "octoai.run/mixtral-8x7b-instruct", + "ttr": 0.09477424621582031, + "ttft": 0.12925434112548828, + "tps": 57.35082102178477, + "num_tokens": 20, + "total_time": 0.46054863929748535, + "output": " Radiant sun descends, a fiery sphere,\nIts brilliant hues of red and" + }, + { + "model": "perplexity.ai/mixtral-8x7b-instruct", + "ttr": 0.06856942176818848, + "ttft": 0.08472752571105957, + "tps": 151.0615701030244, "num_tokens": 20, - "total_time": 0.45353174209594727, - "output": " Radiant sun, in fiery decline,\nDay's last light, a brief reprieve" + "total_time": 0.21050405502319336, + "output": "Radiant sun, in fiery decline,\nYour parting light, a mystic sign." }, { - "model": "api.perplexity.ai/sonar-medium-chat", - "ttr": 0.0875403881072998, - "ttft": 0.09446549415588379, - "tps": 78.66309865883836, + "model": "perplexity.ai/sonar-medium-chat", + "ttr": 0.06742739677429199, + "ttft": 0.0834968090057373, + "tps": 75.50917194748492, "num_tokens": 20, - "total_time": 0.3360018730163574, + "total_time": 0.33512187004089355, "output": "Radiant sun, descending low,\nPainting hues of orange, red.\n" }, { - "model": "fixie-llama-2-70b.westus3.azure", - "ttr": 1.3782143592834473, - "ttft": 1.3784019947052002, + "model": "fireworks.ai/llama-v3-70b-instruct", + "ttr": 0.1935124397277832, + "ttft": 0.20126914978027344, + "tps": 45.54781385082209, + "num_tokens": 20, + "total_time": 0.6184132099151611, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "groq.com/llama3-70b-8192", + "ttr": 0.11503291130065918, + "ttft": 0.23969507217407227, + "tps": 163.8827901199532, + "num_tokens": 20, + "total_time": 0.35563158988952637, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sea\nGolden rays upon" + }, + { + "model": "perplexity.ai/llama-3-70b-instruct", + "ttr": 0.1144716739654541, + "ttft": 0.12107729911804199, + "tps": 41.7589708521648, + "num_tokens": 20, + "total_time": 0.5760693550109863, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "together.xyz/Llama-3-70b-chat-hf", + "ttr": 0.2793278694152832, + "ttft": 0.2793872356414795, + "tps": 95.77221886390649, + "num_tokens": 20, + "total_time": 0.47777462005615234, + "output": "Here is a nonet about a sunset:\n\nVibrant hues upon the sky\nOrange, pink" + }, + { + "model": "azure.westus3/llama-2-70b", + "ttr": 1.2381916046142578, + "ttft": 1.2382867336273193, "tps": 999, "num_tokens": 20, - "total_time": 1.3789358139038086, + "total_time": 1.2385425567626953, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "fixie-llama-2-70b.eastus2.azure", - "ttr": 1.1527869701385498, - "ttft": 1.1530272960662842, + "model": "azure.eastus2/llama-2-70b", + "ttr": 1.2713744640350342, + "ttft": 1.2716302871704102, "tps": 999, "num_tokens": 20, - "total_time": 1.153456687927246, + "total_time": 1.271965742111206, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.fireworks.ai/llama-v2-70b-chat", - "ttr": 0.10815906524658203, - "ttft": 0.14466428756713867, - "tps": 175.0848073538814, + "model": "fireworks.ai/llama-v2-70b-chat", + "ttr": 0.17180705070495605, + "ttft": 0.1923677921295166, + "tps": 113.18857714235394, "num_tokens": 20, - "total_time": 0.25318312644958496, + "total_time": 0.3602292537689209, "output": "Sure, here's a nonet about a sunset:\n\n1. The sky is" }, { - "model": "api.groq.com/llama2-70b-4096", - "ttr": 0.16229581832885742, - "ttft": 0.21788716316223145, - "tps": 236.83051809273212, + "model": "groq.com/llama2-70b-4096", + "ttr": 0.13243913650512695, + "ttft": 0.3076162338256836, + "tps": 188.095924659454, "num_tokens": 21, - "total_time": 0.3023357391357422, + "total_time": 0.41394495964050293, "output": "Sure! Here is a nonet about a sunset:\n\nThe sky is painted orange and and" }, { - "model": "text.octoai.run/llama-2-70b-chat-fp16", - "ttr": 0.017972946166992188, - "ttft": 0.06537008285522461, - "tps": 25.845875227755823, + "model": "octoai.run/llama-2-70b-chat-fp16", + "ttr": 0.015279293060302734, + "ttft": 0.0740976333618164, + "tps": 27.93512073459072, "num_tokens": 20, - "total_time": 0.8004970550537109, + "total_time": 0.7542450428009033, "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "api.perplexity.ai/pplx-70b-chat", - "ttr": 0.14553499221801758, - "ttft": 0.15407085418701172, - "tps": 52.324048631588205, + "model": "together.xyz/llama-2-70b-chat", + "ttr": 0.25617337226867676, + "ttft": 0.2562408447265625, + "tps": 73.89583043561312, "num_tokens": 20, - "total_time": 0.5171926021575928, - "output": "Golden orb descends,\nA fiery ball in twilight's dance,\nC" + "total_time": 0.513359546661377, + "output": " Sure! Here's a nonet about a sunset:\n\n1. The sky" }, { - "model": "togethercomputer/llama-2-70b-chat", - "ttr": 0.22426724433898926, - "ttft": 0.24251222610473633, - "tps": 54.39949322018163, + "model": "fireworks.ai/llama-v2-13b-chat", + "ttr": 0.17316627502441406, + "ttft": 0.1915881633758545, + "tps": 256.7480677472462, "num_tokens": 20, - "total_time": 0.5917801856994629, - "output": "\n\nA sunset's colors dance and play\nAcross the sky, a fiery" + "total_time": 0.2655906677246094, + "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" }, { - "model": "api.fireworks.ai/llama-v2-13b-chat", - "ttr": 0.16057276725769043, - "ttft": 0.16700148582458496, - "tps": 244.8950130910968, + "model": "together.xyz/llama-2-13b-chat", + "ttr": 0.23114252090454102, + "ttft": 0.23120808601379395, + "tps": 43.18192955904912, "num_tokens": 20, - "total_time": 0.24458575248718262, - "output": "Sure, here's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.6712069511413574, + "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "text.octoai.run/llama-2-13b-chat-fp16", - "ttr": 0.015588521957397461, - "ttft": 0.04411506652832031, - "tps": 56.866805197177634, + "model": "octoai.run/llama-2-13b-chat-fp16", + "ttr": 0.046790361404418945, + "ttft": 0.05956387519836426, + "tps": 56.28220564264386, "num_tokens": 20, - "total_time": 0.37822914123535156, + "total_time": 0.39714837074279785, "output": " Sure! Here is a nonet about a sunset:\n\nThe sun sets slow and" }, { - "model": "togethercomputer/llama-2-13b-chat", - "ttr": 0.4765934944152832, - "ttft": 0.47701025009155273, - "tps": 47.039469420304144, + "model": "fireworks.ai/llama-v3-8b-instruct", + "ttr": 0.173828125, + "ttft": 0.1925356388092041, + "tps": 709.1656077028495, "num_tokens": 20, - "total_time": 0.8809263706207275, - "output": "\n\nHere's a nonet about a sunset:\n\nThe sky is painted with" + "total_time": 0.21932768821716309, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nSoftly fade the day" }, { - "model": "api.fireworks.ai/llama-v2-7b-chat", - "ttr": 0.15024352073669434, - "ttft": 0.15726709365844727, - "tps": 175.91615196132537, + "model": "groq.com/llama3-8b-8192", + "ttr": 0.1824946403503418, + "ttft": 3.535099744796753, + "tps": 999, "num_tokens": 20, - "total_time": 0.2652730941772461, - "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + "total_time": 3.5455799102783203, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "api.perplexity.ai/pplx-7b-chat", - "ttr": 0.06827354431152344, - "ttft": 0.07500839233398438, - "tps": 88.45138168212598, + "model": "perplexity.ai/llama-3-8b-instruct", + "ttr": 0.10976505279541016, + "ttft": 0.12586283683776855, + "tps": 151.55022402081227, "num_tokens": 20, - "total_time": 0.28981566429138184, - "output": "A sunset's fiery hues,\nA canvas painted with the evening's blues" + "total_time": 0.2512338161468506, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "togethercomputer/llama-2-7b-chat", - "ttr": 0.2281172275543213, - "ttft": 0.23213720321655273, - "tps": 96.79343339997425, + "model": "together.xyz/Llama-3-8b-chat-hf", + "ttr": 0.21121788024902344, + "ttft": 0.22036385536193848, + "tps": 198.23923502106976, "num_tokens": 20, - "total_time": 0.42843151092529297, - "output": "\n\n 1. Write a haiku about a sunset.\n\n 2. Write" + "total_time": 0.3162076473236084, + "output": "Here is a nonet about a sunset:\n\nGolden hues upon the sea\nFading light, a" }, { - "model": "@cf/meta/llama-2-7b-chat-fp16", - "ttr": 0.6644198894500732, - "ttft": 0.6645960807800293, - "tps": 19.470048286305015, - "num_tokens": 21, - "total_time": 1.69181489944458, + "model": "fireworks.ai/llama-v2-7b-chat", + "ttr": 0.16304779052734375, + "ttft": 0.17381930351257324, + "tps": 269.73837754407816, + "num_tokens": 20, + "total_time": 0.24425792694091797, + "output": "Of course! Here is a nonet about a sunset:\n\nGolden hues" + }, + { + "model": "together.xyz/llama-2-7b-chat", + "ttr": 0.21961617469787598, + "ttft": 0.21973204612731934, + "tps": 88.61563643601613, + "num_tokens": 20, + "total_time": 0.4341411590576172, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "@cf/meta/llama-2-7b-chat-int8", - "ttr": 0.35280656814575195, - "ttft": 0.35302734375, - "tps": 37.229968040823955, + "model": "cloudflare/llama-2-7b-chat-fp16", + "ttr": 0.5513401031494141, + "ttft": 0.5514194965362549, + "tps": 19.69709805851596, "num_tokens": 21, - "total_time": 0.8902289867401123, + "total_time": 1.5667974948883057, "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" }, { - "model": "Neets-7B", - "ttr": 0.4688851833343506, - "ttft": 0.48283815383911133, - "tps": 73.1967367603106, - "num_tokens": 20, - "total_time": 0.7424125671386719, - "output": " A golden light descends, as day surrenders,\nBathing the sky in h" + "model": "cloudflare/llama-2-7b-chat-int8", + "ttr": 0.29263997077941895, + "ttft": 0.29274797439575195, + "tps": 34.37024411429695, + "num_tokens": 21, + "total_time": 0.8746466636657715, + "output": " Sure! Here is a nonet about a sunset:\n\nGolden hues" } ] }