Add conditional formatting, methodology, sources.

Added custom logic to add color formatting to the columns. Removed less important columns. Added sections for methodology and sources.

website/src/components/BenchmarkDefinitions.astro

@@ -1,19 +1,15 @@
 ---
 import Section from './Section.astro';
-import { ModelDefinition, RegionDefinition, TTRDefinition, TTFTDefinition, TPSDefinition, TokensDefinition, TotalTimeDefinition } from '@/utils/DataGridDefinitions';
+import { ModelDefinition, TTFTDefinition, TPSDefinition, TotalTimeDefinition } from '@/utils/DataGridDefinitions';
 ---
 
 <Section>
-	<div class="text-sm">
-		<p class="text-xl font-bold">Definitions</p>
-		<br />
+	<div class="text-base">
+		<p id="definitions" class="text-xl font-bold mb-4">Definitions</p>
 		<ul class="list-none">
 			<li class="pb-2"><b>{ ModelDefinition.title } → </b>{ ModelDefinition.definition }</li>
-			<li class="pb-2"><b>{ RegionDefinition.title } → </b>{ RegionDefinition.definition }</li>
-			<li class="pb-2"><b>{ TTRDefinition.title } → </b>{ TTRDefinition.definition }</li>
 			<li class="pb-2"><b>{ TTFTDefinition.title } → </b>{ TTFTDefinition.definition }</li>
 			<li class="pb-2"><b>{ TPSDefinition.title } → </b>{ TPSDefinition.definition }</li>
-			<li class="pb-2"><b>{ TokensDefinition.title } → </b>{ TokensDefinition.definition }</li>
 			<li class="pb-2"><b>{ TotalTimeDefinition.title } → </b>{ TotalTimeDefinition.definition }</li>
 		</ul>
 	</div>

website/src/components/DataGrid.astro

@@ -20,35 +20,82 @@ import 'ag-grid-community/styles/ag-theme-quartz.css';
     <label for="cdgRegionSelector">Europe (Paris)</label>
   </div>
   <br />
-  <div id="myGrid" class="ag-theme-quartz text-sm bg-stone-100 font-mono text-gray-950 dark:bg-stone-900 dark:text-white " style="width: 1000px;"></div>
-  <div id="lastUpdated" class="text-base font-mono italic text-right"></div>
+  <div id="myGrid" class="ag-theme-quartz text-sm bg-stone-100 font-mono text-gray-950 dark:bg-stone-900 dark:text-white " style="width: 800px;"></div>
 </form>
 
 <script>
   import { createGrid } from 'ag-grid-community';
-  import { BenchmarkRegions, gridOptions, LatestDataUrl } from '@/utils/DataGridDefinitions.ts';
+  import { BenchmarkRegions, gridOptions } from '@/utils/DataGridDefinitions.ts';
+  import { generateColor } from '@/utils/DataGridColors.js';
   import { fetchLocalJsonFile } from '@/utils/FetchData.ts';
 
   let gridApi;
   const localData = '../../data/latest.json';
   let gridData = [];
-
+  let selectedRegion = 'sea';
+  let selectedRegionData = [];
+  const startColor = '#FF7F00';  // #FF0000 red
+  const endColor = '#377EB8'; // #00FF00 green
+
   function setGridData(region, gridData) {
+    selectedRegion = region;
     gridData.forEach((regionData) => {
       if (regionData.region === region) {
-        gridApi.setGridOption("rowData", regionData.results);
+        selectedRegionData = regionData.results;
+        gridApi.setGridOption("rowData", selectedRegionData);
+      }
+    });
+  };
+
+  // Returns the background color for the cell based on the value
+  function colFunction(params) {
+    const currentColumnID = params.column.colId;
+    const totalCellCount = params.api.getDisplayedRowCount();
+    let allValuesInCurrentCol = [];
+
+    selectedRegionData.forEach((x) => {
+      allValuesInCurrentCol = allValuesInCurrentCol.concat(x[currentColumnID]);
+    });
+
+    const valuesForTableOrdered = allValuesInCurrentCol.sort(function (a, b) {
+      // TTFT and Total Time: lower number is better
+      if (currentColumnID === 'ttft' || currentColumnID === 'total_time') {
+        return a - b;
+      } else {
+        return b - a;
       }
     });
-  } 
+    const valueIndex = valuesForTableOrdered.indexOf(params.value);
+    const bgColour = generateColor(startColor, endColor, totalCellCount, valueIndex)
+
+    return { backgroundColor: '#' + bgColour };
+  };
+
+  function setCellFormatter() {
+    const columnDefs = gridApi.getColumnDefs();
+    columnDefs.forEach((colDef, index) => {
+      // Only apply the formatter for ttft, tps
+      if(colDef.colId == 'ttft' || colDef.colId == 'tps' || colDef.colId == 'total_time') {
+        colDef.cellStyle = colFunction;
+      }      
+    });
+    gridApi.setGridOption("columnDefs", columnDefs);
+  };
 
   async function onDOMContentLoaded() {
     gridData = await fetchLocalJsonFile(localData);
 
     // setup the grid after the page has finished loading
     var gridDiv = document.querySelector("#myGrid");
     gridApi = createGrid(gridDiv, gridOptions);
-    setGridData('sea', gridData);   // default to sea region on load
-    document.getElementById('lastUpdated').innerText = `Last Updated: ${gridData[0].time.split('T')[0]}`;
+    setGridData(selectedRegion, gridData);   // default to sea region on load
+    setCellFormatter();
+
+    // Add text for our last updated date
+    const newDiv = document.createElement('div');
+    newDiv.classList.add('text-base', 'font-mono', 'italic', 'text-right');
+    newDiv.innerText = `Last Updated: ${gridData[0].time.split('T')[0]}`;
+    gridDiv.appendChild(newDiv);
 
     // Add an event listener to the radio buttons to filter the grid data
     document.getElementById('benchmarks').addEventListener('change', function(event) {

website/src/components/Hero.astro

@@ -3,12 +3,9 @@ import Section from './Section.astro';
 ---
 
 <Section>
-	<p class="pb-2 pt-6 text-4xl font-bold">Slow LLMs Suck</p>
-	<div class="text-sm">
-		<p class="py-2">Apps are only as fast as their slowest component. LLMs vary tremendously on quality and speed. The table below shows the lastest performance metrics for many models and environments. We hope this data helps you choose the most performant model. </p>
-		<br />
-		<p>Have another model you want us to benchmark? Hit us up with an <a class="underline hover:text-orange-600" href="https://github.com/fixie-ai/fastest.ai/issues" target="_blank">issue on GitHub.</a></p>
-	</div>
+	<p>The speed of various LLMs varies tremendously. This site provides benchmarks for the top models. We hope this data will help you choose the best model for your needs!</p>
+	<br />
+	<p>Have another model you want us to benchmark? Hit us up with an <a class="underline hover:text-orange-600" href="https://github.com/fixie-ai/fastest.ai/issues" target="_blank">issue on GitHub.</a></p>
 </Section>
 
 

website/src/components/HowToUse.astro

@@ -3,10 +3,10 @@ import Section from './Section.astro';
 ---
 
 <Section>
-	<div class="text-sm">
-		<p class="text-xl font-bold">How to Use</p>
+	<div class="text-base">
+		<p class="text-xl font-bold mb-4">How to Use</p>
+		<p>The table below is interactive. You can sort on any column. Further, the hamburger menu (appears when you hover a column header) allows for custom filtering. <a class="underline hover:text-orange-600" href="#definitions">Definitions</a>, <a class="underline hover:text-orange-600" href="#methodology">methodology</a>, and links to <a class="underline hover:text-orange-600" href="#source">source</a> below.</p>
 		<br />
-		<p>The above table is interactive. You can sort on any column. Further, the hamburger menu (appears when you hover a column header) allows for custom filtering. Have fun exploring!</p>
-
+		<p>Have fun exploring!</p>
 	</div>
 </Section>

website/src/components/Methodology.astro

@@ -0,0 +1,17 @@
+---
+import Section from './Section.astro';
+---
+
+<Section>
+	<div class="text-base">
+		<p id="methodology" class="text-xl font-bold mb-4">Methodology</p>
+		<p class="py-2"><b>Distributed Footprint → </b>We run our tools daily in multiple data centers using <a class="underline hover:text-orange-600"  href="https://fly.io/docs/reference/regions/" target="_blank">Fly.io</a>. Currently we run in cdg, iad, and sea.</p>
+		<p class="py-2"><b>Short Requests → </b>Input requests are a short, single message (~20 tokens) and ask for a brief output response.</p>
+		<p class="py-2"><b>Max Output → </b>Max output tokens is set to 100 to avoid distortion of TPS values from long outputs.</p>
+		<p class="py-2"><b>Warmup for Consistent Setup → </b>A warmup connection is made to remove any connection setup latency.</p>
+		<p class="py-2"><b>TTFT Roundtrip → </b>The TTFT clock starts when the HTTP request is made and stops when the first token result is received in the response stream.</p>
+		<p class="py-2"><b>Try 3, Keep 1 → </b>For each provider, three separate inferences are done, and the best result is kept (to remove any outliers due to queuing etc).</p>
+	</div>
+</Section>
+
+

website/src/components/Sources.astro

@@ -0,0 +1,14 @@
+---
+import Section from './Section.astro';
+---
+
+<Section>
+	<div class="text-base">
+		<p id="source" class="text-xl font-bold">Source</p>
+		<p class="py-2"><b>Raw Data → </b>All data is in this public <a class="underline hover:text-orange-600" href="https://storage.googleapis.com/thefastest-data" target="_blank">GCS bucket.</a></p>
+		<p class="py-2"><b>Benchmarking Tools → </b>The full test suite is available in the <a class="underline hover:text-orange-600" href="https://github.com/fixie-ai/ai-benchmarks" target="_blank">ai-benchmarks repo.</a></p>
+		<p class="py-2"><b>Website → </b>Full source code for this site is on <a class="underline hover:text-orange-600" href="https://github.com/fixie-ai/fastest.ai" target="_blank">GitHub.</a></p>
+	</div>
+</Section>
+
+

website/src/pages/index.astro

@@ -6,16 +6,21 @@ import Section from '@/components/Section.astro';
 import DataGrid from '@/components/DataGrid.astro';
 import BenchmarkDefinitions from '@/components/BenchmarkDefinitions.astro';
 import HowToUse from '@/components/HowToUse.astro';
+import Methodology from '@/components/Methodology.astro';
+import Sources from '@/components/Sources.astro';
 
 const { title } = AppConfig;
 const { description } = AppConfig;
 
 ---
 <Base head={{ title, description }}>
 	<Hero />
+	<HowToUse />
 	<Section>
 		<DataGrid />
 	</Section>
-	<HowToUse />
+
 	<BenchmarkDefinitions />
+	<Methodology />
+	<Sources />
 </Base>

website/src/utils/AppConfig.ts

@@ -1,6 +1,6 @@
 export const AppConfig = {
 	site_name: 'The Fastest.AI',
-	title: 'Fastest.ai',
+	title: 'TheFastest.ai',
 	description: 'Benchmarks for the fastest AI models',
 	author: 'Fixie.ai',
 	locale_region: 'en-us',

website/src/utils/DataGridColors.js

@@ -0,0 +1,56 @@
+function hex (c) {
+  var s = "0123456789abcdef";
+  var i = parseInt (c);
+  if (i == 0 || isNaN (c))
+    return "00";
+  i = Math.round (Math.min (Math.max (0, i), 255));
+  return s.charAt ((i - i % 16) / 16) + s.charAt (i % 16);
+}
+
+/* Convert an RGB triplet to a hex string */
+function convertToHex (rgb) {
+  return hex(rgb[0]) + hex(rgb[1]) + hex(rgb[2]);
+}
+
+/* Remove '#' in color hex string */
+function trim (s) { return (s.charAt(0) == '#') ? s.substring(1, 7) : s }
+
+/* Convert a hex string to an RGB triplet */
+function convertToRGB (hex) {
+  var color = [];
+  color[0] = parseInt ((trim(hex)).substring (0, 2), 16);
+  color[1] = parseInt ((trim(hex)).substring (2, 4), 16);
+  color[2] = parseInt ((trim(hex)).substring (4, 6), 16);
+  return color;
+}
+
+export function generateColor(colorStart,colorEnd,colorCount,index){
+  // The beginning of your gradient
+  var start = convertToRGB (colorStart);    
+
+  // The end of your gradient
+  var end   = convertToRGB (colorEnd);    
+
+  // The number of colors to compute
+  var len = colorCount;
+
+  //Alpha blending amount
+  var alpha = 0.0;
+
+  var saida = [];
+
+  for (let i = 0; i < len; i++) {
+    var c = [];
+    alpha += (1.0/len);
+
+    c[0] = start[0] * alpha + (1 - alpha) * end[0];
+    c[1] = start[1] * alpha + (1 - alpha) * end[1];
+    c[2] = start[2] * alpha + (1 - alpha) * end[2];
+
+    saida.push(convertToHex (c));
+
+  }
+
+  return saida[index];
+
+}

website/src/utils/DataGridDefinitions.ts

@@ -8,24 +8,19 @@ export const BenchmarkRegions = [
 		region: 'iad',
 		label: 'US East (Virginia)',
 		dateAdded: '2024-04-13',
-		latestData: 'https://storage.googleapis.com/thefastest-data/iad/text/2024-04-13.json',
 	},
 	{
 		region: 'cdg',
 		label: 'Europe (Paris)',
 		dateAdded: '2024-04-13',
-		latestData: 'https://storage.googleapis.com/thefastest-data/cdg/text/2024-04-13.json',
 	},
 	{
 		region: 'sea',
 		label: 'US West (Seattle)',
 		dateAdded: '2024-04-13',
-		latestData: 'https://storage.googleapis.com/thefastest-data/sea/text/2024-04-13.json',
 	},
 ];
 
-export const LatestDataUrl = 'https://storage.googleapis.com/thefastest-data/latest/text/latest.json';
-
 export const ModelDefinition = {
 	title: "Provider/Model",
 	definition: "The LLM and host provider used."
@@ -71,7 +66,7 @@ const columnModel = {
 	headerTooltip: ModelDefinition.definition,
 	headerClass: headerClass,
 	minWidth: 400, 
-	tooltipField: "output"
+	// tooltipField: "output"
 };
 
 // Region column
@@ -119,7 +114,7 @@ const columnNumTokens = {
 	headerName: TokensDefinition.title,
 	headerTooltip: TokensDefinition.definition,
 	headerClass: headerClass,
-	minWidth: 100,
+	// minWidth: 100,
 	maxWidth: 100,
 	wrapHeaderText: true
 };
@@ -130,21 +125,23 @@ const columnTotalTime = {
 	headerName: TotalTimeDefinition.title,
 	headerTooltip: TotalTimeDefinition.definition,
 	headerClass: headerClass,
-	minWidth: 100,
+	// minWidth: 100,
 	maxWidth: 100,
 	wrapHeaderText: true,
-	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2)
+	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2),
 };
 
 export const gridOptions = {
-  alwaysShowVerticalScroll: true,
-  autoSizeStrategy: { type: 'fitCellContents' },
+  // alwaysShowVerticalScroll: true,
+  autoSizeStrategy: { type: 'fitGridWidth' },
+	enableCellTextSelection: true,
   defaultColDef: {
     filter: true,
-    minWidth: 80,
+		resizable: false,
+    // minWidth: 80,	
   },
 	domLayout: 'autoHeight',
   rowData: [],
-  // Columns to be displayed (Should match rowData properties)...omit columnRegion
-  columnDefs: [ columnModel, columnTTR, columnTTFT, columnTPS, columnNumTokens, columnTotalTime]
+  // Columns to be displayed (Should match rowData properties)...omit columnRegion, columnTTR, columnNumTokens
+  columnDefs: [ columnModel, columnTTFT, columnTPS, columnTotalTime]
 };