Skip to content

Commit 2ce0433

Browse files
watsonwatson
committed
WIP: Benchmark
1 parent f341bd3 commit 2ce0433

9 files changed

+607
-0
lines changed

‎test/benchmarks/001-custom-transactions.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
'use strict'
2+
3+
process.on('SIGINT', end)
4+
5+
let agent
6+
if (process.env.AGENT) {
7+
agent = require('../../').start({
8+
serviceName: '001-custom-transactions',
9+
captureExceptions: false,
10+
captureSpanStackTraces: false
11+
})
12+
}
13+
14+
let start
15+
let stop = false
16+
const pid = process.argv[2]
17+
const warmup = 1e4
18+
const runtime = 10
19+
const metrics = {
20+
transactions: 0
21+
}
22+
23+
// warmup
24+
console.error('Warming up for %d transactions...', warmup)
25+
addTransaction('warmup', function runAgain () {
26+
if (metrics.transactions < warmup) return addTransaction('warmup', runAgain)
27+
28+
console.error('Running benchmark for %d seconds...', runtime)
29+
30+
setTimeout(end, runtime * 1000)
31+
32+
// process.kill(pid, 'SIGINT')
33+
metrics.transactions = 0
34+
start = process.hrtime()
35+
36+
// actual benchmark
37+
addTransaction('benchmark', function runAgain () {
38+
addTransaction('benchmark', runAgain)
39+
})
40+
})
41+
42+
function addTransaction (name, cb) {
43+
if (stop) return
44+
if (agent) agent.startTransaction(name)
45+
addSpan(name, () => {
46+
if (agent) agent.endTransaction()
47+
metrics.transactions++
48+
setImmediate(cb)
49+
})
50+
}
51+
52+
function addSpan (prefix, amount, cb) {
53+
if (typeof amount === 'function') return addSpan(prefix, 1, amount)
54+
setImmediate(() => {
55+
const span = agent && agent.startSpan(prefix + 'my-span', 'my-span-type')
56+
setImmediate(() => {
57+
if (agent) span.end()
58+
if (--amount === 0) cb()
59+
else addSpan(prefix, amount, cb)
60+
})
61+
})
62+
}
63+
64+
function end () {
65+
const hrtime = process.hrtime(start)
66+
stop = true
67+
console.error('Flushing...')
68+
agent.flush(function () {
69+
setTimeout(function () {
70+
process.kill(pid, 'SIGINT')
71+
process.stdout.write(JSON.stringify({hrtime, metrics}))
72+
process.exit()
73+
}, 100)
74+
})
75+
}

‎test/benchmarks/002-custom-transactions-normal-sized.js‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
'use strict'
2+
3+
process.on('SIGINT', end)
4+
5+
let agent
6+
if (process.env.AGENT) {
7+
agent = require('../../').start({
8+
serviceName: '002-custom-transactions-normal-sized',
9+
captureExceptions: false
10+
})
11+
}
12+
13+
const callstack = require('./utils/callstack')
14+
15+
let start
16+
const pid = process.argv[2]
17+
const warmup = 1e4
18+
const runtime = 20
19+
const metrics = {
20+
transactions: 0
21+
}
22+
23+
// To avoid randomness, but still generate what appears to be natural random
24+
// call stacks, number of spans etc, use a pre-defined set of numbers
25+
const numbers = [2, 5, 10, 0, 2, 21, 2, 5, 6, 9, 1, 11, 9, 8, 12]
26+
let numbersSpanIndex = 5
27+
let numbersStackLevelIndex = 0
28+
29+
// warm up
30+
console.error('Warming up for %d transactions...', warmup)
31+
addTransaction(function runAgain () {
32+
if (metrics.transactions < warmup) return addTransaction(runAgain)
33+
34+
console.error('Running benchmark for %d seconds...', runtime)
35+
36+
setTimeout(end, runtime * 1000)
37+
38+
process.kill(pid, 'SIGINT')
39+
metrics.transactions = 0
40+
start = process.hrtime()
41+
42+
// actual benchmark
43+
addTransaction(function runAgain () {
44+
addTransaction(runAgain)
45+
})
46+
})
47+
48+
function addTransaction (cb) {
49+
if (agent) agent.startTransaction('my-transaction')
50+
const amount = numbers[numbersStackLevelIndex % numbers.length]
51+
callstack(amount, () => {
52+
const amount = numbers[numbersSpanIndex % numbers.length]
53+
addSpan(amount, () => {
54+
if (agent) agent.endTransaction()
55+
metrics.transactions++
56+
setImmediate(cb)
57+
})
58+
})
59+
}
60+
61+
function addSpan (amount, cb) {
62+
if (typeof amount === 'function') return addSpan(1, amount)
63+
// setImmediate(() => {
64+
const span = agent && agent.startSpan('my-span', 'my-span-type')
65+
// setImmediate(() => {
66+
if (agent) span.end()
67+
if (--amount === 0) cb()
68+
else addSpan(amount, cb)
69+
// })
70+
// })
71+
}
72+
73+
function end () {
74+
const hrtime = process.hrtime(start)
75+
process.kill(pid, 'SIGINT')
76+
process.stdout.write(JSON.stringify({hrtime, metrics}))
77+
process.exit()
78+
}

‎test/benchmarks/003-span-creation-overhead.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
'use strict'
2+
3+
process.on('SIGINT', end)
4+
5+
let agent
6+
if (process.env.AGENT) {
7+
agent = require('../../').start({
8+
serviceName: '003-span-creation-overhead',
9+
captureExceptions: false
10+
})
11+
// v1 no-ops
12+
if (agent._instrumentation._queue) {
13+
agent._instrumentation._queue.add = noop
14+
}
15+
// v2 no-ops
16+
if (agent._apmServer) {
17+
agent._apmServer.sendTransaction = noop
18+
agent._apmServer.sendSpan = noop
19+
agent._apmServer.sendError = noop
20+
}
21+
}
22+
23+
let start
24+
const warmup = 1e4
25+
const runtime = 10
26+
const metrics = {
27+
transactions: 0
28+
}
29+
30+
// warmup
31+
console.error('Warming up for %d transactions...', warmup)
32+
addTransaction('warmup', function runAgain () {
33+
if (metrics.transactions < warmup) return addTransaction('warmup', runAgain)
34+
35+
console.error('Running benchmark for %d seconds...', runtime)
36+
37+
setTimeout(end, runtime * 1000)
38+
39+
metrics.transactions = 0
40+
start = process.hrtime()
41+
42+
// actual benchmark
43+
addTransaction('benchmark', function runAgain () {
44+
addTransaction('benchmark', runAgain)
45+
})
46+
})
47+
48+
function addTransaction (name, cb) {
49+
if (agent) agent.startTransaction(name)
50+
addSpan(name, () => {
51+
if (agent) agent.endTransaction()
52+
metrics.transactions++
53+
setImmediate(cb)
54+
})
55+
}
56+
57+
function addSpan (prefix, amount, cb) {
58+
if (typeof amount === 'function') return addSpan(prefix, 1, amount)
59+
setImmediate(() => {
60+
const span = agent && agent.startSpan(prefix + 'my-span', 'my-span-type')
61+
setImmediate(() => {
62+
if (agent) span.end()
63+
if (--amount === 0) cb()
64+
else addSpan(prefix, amount, cb)
65+
})
66+
})
67+
}
68+
69+
function end () {
70+
const hrtime = process.hrtime(start)
71+
process.stdout.write(JSON.stringify({hrtime, metrics}))
72+
process.exit()
73+
}
74+
75+
function noop () {}

‎test/benchmarks/_client.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
'use strict'
2+
3+
const http = require('http')
4+
5+
const start = Date.now()
6+
let requests = 0
7+
8+
request()
9+
10+
function request (n) {
11+
if (!n) n = 1
12+
else if (n === 1e4) return done()
13+
14+
const req = http.request({
15+
socketPath: '/tmp/benchmark-app.sock'
16+
}, function (res) {
17+
res.resume()
18+
res.on('end', function () {
19+
requests++
20+
request(++n)
21+
})
22+
})
23+
req.end()
24+
}
25+
26+
function done () {
27+
const seconds = (Date.now() - start) / 1000
28+
process.stdout.write(`${requests / seconds}`)
29+
}

‎test/benchmarks/utils/analyzer.js‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
'use strict'
2+
3+
const fs = require('fs')
4+
const os = require('os')
5+
const joinPath = require('path').join
6+
const numeral = require('numeral')
7+
const columnify = require('columnify')
8+
const git = require('git-rev')
9+
10+
const args = process.argv.slice(2)
11+
const appfile = args[0]
12+
const serverfile = args[1]
13+
14+
const applog = JSON.parse(fs.readFileSync(appfile))
15+
const serverlog = fs.existsSync(serverfile) ? JSON.parse(fs.readFileSync(serverfile)) : null
16+
17+
const data = serverlog
18+
? [].concat(
19+
output(applog),
20+
output(serverlog)
21+
)
22+
: output(applog)
23+
24+
displayResult()
25+
storeResult()
26+
27+
function displayResult () {
28+
console.log()
29+
console.log('Benchmark running time: %d seconds', applog.seconds)
30+
if (serverlog) console.log('Server running time: %d seconds', serverlog.seconds)
31+
if (serverlog) console.log('Avg bytes per event:', format(serverlog.metrics.bytes / serverlog.metrics.events))
32+
33+
const opts = {
34+
config: {
35+
rate: {align: 'right'},
36+
total: {align: 'right'}
37+
}
38+
}
39+
console.log(`\n${columnify(data, opts)}\n`)
40+
}
41+
42+
function storeResult () {
43+
git.short(function (rev) {
44+
const file = joinPath(__dirname, '..', '.tmp', 'result.json')
45+
46+
const json = JSON.stringify({
47+
os: {
48+
arch: os.arch(),
49+
cpus: os.cpus(),
50+
freemem: os.freemem(),
51+
homedir: os.homedir(),
52+
hostname: os.hostname(),
53+
loadavg: os.loadavg(),
54+
platform: os.platform(),
55+
release: os.release(),
56+
totalmem: os.totalmem(),
57+
type: os.type(),
58+
uptime: os.uptime()
59+
},
60+
process: {
61+
arch: process.arch(),
62+
config: process.config,
63+
env: process.env,
64+
platform: process.platform,
65+
release: process.release,
66+
version: process.version,
67+
versions: process.versions
68+
},
69+
repo: {
70+
rev
71+
},
72+
results: data
73+
})
74+
75+
fs.writeFile(file, json, function (err) {
76+
if (err) throw err
77+
console.log()
78+
console.log('Stored Elasticsearch result document at:', file)
79+
})
80+
})
81+
}
82+
83+
function output (log) {
84+
if (!log) return
85+
const hrtime = log.hrtime
86+
log.seconds = (hrtime[0] * 1e9 + hrtime[1]) / 1e9
87+
return Object.entries(log.metrics).map(([metric, count]) => {
88+
return {
89+
metric,
90+
rate: `${format(count / log.seconds)}/s`,
91+
total: format(count, false)
92+
}
93+
})
94+
}
95+
96+
function format (n, decimals) {
97+
if (decimals === undefined) decimals = true
98+
return numeral(n).format(decimals ? '0,0.00' : '0,0')
99+
}

0 commit comments

Comments
 (0)