feat: save/compare results to/from file

Author: mistakia

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 .nyc_output
+test/tmp

package-lock.json

@@ -13,6 +13,7 @@
   "license": "MIT",
   "dependencies": {
     "expose-gc": "^1.0.0",
+    "fs-extra": "^9.1.0",
     "yargs": "^15.4.1"
   },
   "localMaintainers": [

package.json

@@ -9,6 +9,18 @@ const argv = yargs
   .version(false)
   .help('help').alias('help', 'h')
   .options({
+    save: {
+      alias: 's',
+      description: 'Save results to file',
+      requiresArg: true,
+      required: false,
+    },
+    compare: {
+      alias: 'c',
+      description: 'Compare results to file',
+      requiresArg: true,
+      required: false,
+    },
     baseline: {
       alias: 'b',
       description: 'Run baseline benchmarks only',

src/cli.js

@@ -2,6 +2,7 @@ require('expose-gc')
 global.gc()
 
 const os = require('os')
+const fs = require('fs-extra')
 
 const report = require('./report')
 
@@ -52,8 +53,14 @@ const start = async (benchmarks, argv) => {
     output += ` in ${(runnerElapsed / 1000000000).toFixed(2)} seconds`
     process.stdout.write(output)
 
+    if (argv.save) {
+      fs.ensureFileSync(argv.save)
+      fs.writeJsonSync(argv.save, results, { spaces: 2 })
+    }
+
     if (argv.report) {
-      report(results)
+      const compare = argv.compare ? fs.readJsonSync(argv.compare) : undefined
+      report(results, compare)
     }
   } catch (e) {
     console.log(e)

src/index.js

@@ -9,7 +9,7 @@ const padStr = (str, max) => {
   return str + repeatStr(' ', length)
 }
 
-const reporter = (results) => {
+const reporter = (results, compare = []) => {
   const reports = [{
     name: 'Benchmark Name',
     ops: 'Ops / ms',
@@ -22,34 +22,51 @@ const reporter = (results) => {
   let maxMemWidth = reports[0].mem.length
 
   for (const benchmark of results) {
+    const savedBenchmark = compare.find(c => c.name === benchmark.name)
+
     const nameWidth = benchmark.name.length
     if (maxNameWidth < nameWidth) {
       maxNameWidth = nameWidth
     }
 
-    const totalMs = (benchmark.elapsed / 1000000).toFixed(4)
-    const opsPerMs = (benchmark.stats.count / totalMs).toFixed(4)
-    const memUsed = benchmark.memory.after.heapUsed - benchmark.memory.before.heapUsed
-    const memUsedMb = (memUsed / 1024 / 1024).toFixed(2)
+    const getCalculations = (b) => {
+      const totalMs = (b.elapsed / 1000000).toFixed(4)
+      const opsPerMs = (b.stats.count / totalMs).toFixed(4)
+      const memUsed = b.memory.after.heapUsed - b.memory.before.heapUsed
+      const memUsedMb = (memUsed / 1024 / 1024).toFixed(2)
+      return { totalMs, opsPerMs, memUsed, memUsedMb }
+    }
+
+    const calculations = getCalculations(benchmark)
+    const savedCalculations = savedBenchmark ? getCalculations(savedBenchmark) : undefined
+    const getOutput = (type) => {
+      const delta = savedCalculations ? (calculations[type] - savedCalculations[type]).toFixed(2) : 0
+      return delta ? `${calculations[type]} (${delta})` : calculations[type]
+    }
 
-    const opsWidth = opsPerMs.toString().length
+    const ops = getOutput('opsPerMs')
+    const opsWidth = ops.toString().length
     if (maxOpsWidth < opsWidth) {
       maxOpsWidth = opsWidth
     }
+
+    const totalMs = getOutput('totalMs')
     const totalWidth = totalMs.toString().length
     if (maxTotalWidth < totalWidth) {
       maxTotalWidth = totalWidth
     }
-    const memWidth = memUsedMb.toString().length
+
+    const mem = getOutput('memUsedMb')
+    const memWidth = mem.toString().length
     if (maxMemWidth < memWidth) {
       maxMemWidth = memWidth
     }
 
     reports.push({
       name: benchmark.name,
-      ops: opsPerMs,
-      totalMs: totalMs,
-      mem: memUsedMb
+      ops,
+      totalMs,
+      mem
     })
   }
 

src/report.js

@@ -24,6 +24,8 @@ describe('CLI Test', () => {
     expect(output).to.contain('stressLimit')
     expect(output).to.contain('baselineLimit')
     expect(output).to.contain('logLimit')
+    expect(output).to.contain('save')
+    expect(output).to.contain('compare')
   })
 
   describe('baseline', () => {
@@ -270,4 +272,5 @@ describe('CLI Test', () => {
       await start(benchmarks, yargsResult)
     })
   })
+
 })

test/cli.test.js

@@ -0,0 +1,31 @@
+/* eslint-disable no-unused-expressions */
+
+const yargs = require('yargs')
+const path = require('path')
+const fs = require('fs-extra')
+const cli = require('../src/cli')
+const expect = require('chai').expect
+const { start } = require('../src/index')
+
+const benchmarks = require(process.cwd() + '/test/benchmarks')
+
+describe('CLI Compare', function () {
+  this.timeout(6000)
+
+  it('should compare results to file', async () => {
+    const yargsCmd = yargs.command(cli)
+    const tmpFile = path.join(__dirname, 'tmp/benchmark.json')
+    const yargsResult1 = yargsCmd.parse(
+      `cli --baseline -s ${tmpFile}`, {}
+    )
+
+    await start(benchmarks, yargsResult1)
+
+    const yargsResult2 = yargsCmd.parse(
+      `cli --baseline -r -c ${tmpFile}`, {}
+    )
+    await start(benchmarks, yargsResult2)
+
+    // TODO - test reporter output
+  })
+})

test/compare.test.js

@@ -0,0 +1,41 @@
+/* eslint-disable no-unused-expressions */
+
+const yargs = require('yargs')
+const path = require('path')
+const fs = require('fs-extra')
+const cli = require('../src/cli')
+const expect = require('chai').expect
+const { start } = require('../src/index')
+
+const benchmarks = require(process.cwd() + '/test/benchmarks')
+
+describe('CLI Save', function () {
+  this.timeout(6000)
+
+  it('should save results to file', async () => {
+    const yargsCmd = yargs.command(cli)
+    const tmpFile = path.join(__dirname, 'tmp/benchmark.json')
+    const yargsResult = yargsCmd.parse(
+      `cli --baseline -s ${tmpFile}`, {}
+    )
+
+    await start(benchmarks, yargsResult)
+
+    const results = fs.readJsonSync(tmpFile)
+    expect(results).to.be.an('array')
+    expect(results.length).to.be.equal(1)
+
+    const benchmark = results[0]
+    expect(benchmark).to.have.property('name')
+    expect(benchmark).to.have.property('cpus')
+    expect(benchmark).to.have.property('loadavg')
+    expect(benchmark).to.have.property('elapsed')
+    expect(benchmark).to.have.property('stats')
+    expect(benchmark).to.have.property('memory')
+
+    expect(benchmark.name).to.be.equal('console-baseline')
+    expect(benchmark.memory).to.have.property('before')
+    expect(benchmark.memory).to.have.property('after')
+    expect(benchmark.stats).to.have.property('count')
+  })
+})