Skip to content

Commit 312885d

Browse files
akvladakvlad
authored
Merge pull request #577 from metrico/fix/profiles_574_576
WIP: fix/profiles 574 576
2 parents 8c73c48 + 0881437 commit 312885d

File tree

5 files changed

+201
-39
lines changed

5 files changed

+201
-39
lines changed

‎lib/bun_wrapper.js

+11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const { Transform } = require('stream')
22
const log = require('./logger')
33
const { EventEmitter } = require('events')
4+
const zlib = require('zlib')
45

56
class BodyStream extends Transform {
67
_transform (chunk, encoding, callback) {
@@ -121,6 +122,16 @@ const wrapper = (handler, parsers) => {
121122
headers['Content-Type'] = 'application/json'
122123
response = JSON.stringify(response)
123124
}
125+
if (response && (ctx.headers.get('accept-encoding') || '').indexOf('gzip') !== -1) {
126+
if (response.on) {
127+
const _r = zlib.createGzip()
128+
response.pipe(_r)
129+
response = _r
130+
} else {
131+
response = Bun.gzipSync(response)
132+
}
133+
headers['Content-Encoding'] = 'gzip'
134+
}
124135
return new Response(response, { status: status, headers: headers })
125136
}
126137
return res

‎pyroscope/json_parsers.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ const labelNames = async (req, payload) => {
3333
return {
3434
getStart: () => body.start,
3535
getEnd: () => body.end,
36-
getName: () => body.name
36+
getName: () => body.name,
37+
getMatchersList: () => body.matchers
3738
}
3839
}
3940

@@ -43,7 +44,7 @@ const labelValues = async (req, payload) => {
4344
body = JSON.parse(body.toString())
4445
return {
4546
getName: () => body.name,
46-
getMatchers: () => body.matchers,
47+
getMatchersList: () => body.matchers,
4748
getStart: () => body.start,
4849
getEnd: () => body.end
4950
}

‎pyroscope/pyroscope.js

+184-34
Original file line numberDiff line numberDiff line change
@@ -53,24 +53,60 @@ WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND date <= toDa
5353
}
5454

5555
const labelNames = async (req, res) => {
56+
const body = req.body
5657
const dist = clusterName ? '_dist' : ''
5758
const fromTimeSec = Math.floor(req.body && req.body.getStart
5859
? parseInt(req.body.getStart()) / 1000
5960
: (Date.now() - HISTORY_TIMESPAN) / 1000)
6061
const toTimeSec = Math.floor(req.body && req.body.getEnd
6162
? parseInt(req.body.getEnd()) / 1000
6263
: Date.now() / 1000)
63-
const labelNames = await clickhouse.rawRequest(`SELECT DISTINCT key
64-
FROM profiles_series_keys${dist}
65-
WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON`,
66-
null, DATABASE_NAME())
64+
if (!body.getMatchersList || body.getMatchersList().length === 0) {
65+
const q = `SELECT DISTINCT key
66+
FROM profiles_series_keys ${dist}
67+
WHERE date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))
68+
AND date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON`
69+
console.log(q)
70+
const labelNames = await clickhouse.rawRequest(q, null, DATABASE_NAME())
71+
const resp = new types.LabelNamesResponse()
72+
resp.setNamesList(labelNames.data.data.map(label => label.key))
73+
return resp
74+
}
75+
const promises = []
76+
for (const matcher of body.getMatchersList()) {
77+
const specialMatchers = getSpecialMatchers(matcher)
78+
const idxReq = matcherIdxRequest(matcher, specialMatchers, fromTimeSec, toTimeSec)
79+
const withIdxReq = new Sql.With('idx', idxReq)
80+
const specialClauses = specialMatchersQuery(specialMatchers.matchers,
81+
'sample_types_units')
82+
const serviceNameSelector = serviceNameSelectorQuery(matcher)
83+
const req = (new Sql.Select()).with(withIdxReq)
84+
.select('key')
85+
.distinct(true)
86+
.from(`profiles_series_gin${dist}`)
87+
.where(Sql.And(
88+
specialClauses,
89+
serviceNameSelector,
90+
Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)),
91+
Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)),
92+
new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxReq))
93+
))
94+
promises.push(clickhouse.rawRequest(req.toString() + ' FORMAT JSON', null, DATABASE_NAME()))
95+
}
96+
const labelNames = await Promise.all(promises)
97+
const labelNamesDedup = Object.fromEntries(
98+
labelNames.flatMap(val => {
99+
return val.data.data.map(row => [row.key, true])
100+
})
101+
)
67102
const resp = new types.LabelNamesResponse()
68-
resp.setNamesList(labelNames.data.data.map(label => label.key))
103+
resp.setNamesList([...Object.keys(labelNamesDedup)])
69104
return resp
70105
}
71106

72107
const labelValues = async (req, res) => {
73108
const dist = clusterName ? '_dist' : ''
109+
const body = req.body;
74110
const name = req.body && req.body.getName
75111
? req.body.getName()
76112
: ''
@@ -83,13 +119,45 @@ const labelValues = async (req, res) => {
83119
if (!name) {
84120
throw new Error('No name provided')
85121
}
86-
const labelValues = await clickhouse.rawRequest(`SELECT DISTINCT val
122+
if (!body.getMatchersList || body.getMatchersList().length === 0) {
123+
const labelValues = await clickhouse.rawRequest(`SELECT DISTINCT val
87124
FROM profiles_series_gin${dist}
88125
WHERE key = ${Sql.quoteVal(name)} AND
89126
date >= toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)})) AND
90127
date <= toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)})) FORMAT JSON`, null, DATABASE_NAME())
128+
const resp = new types.LabelValuesResponse()
129+
resp.setNamesList(labelValues.data.data.map(label => label.val))
130+
return resp
131+
}
132+
const promises = []
133+
for (const matcher of body.getMatchersList()) {
134+
const specialMatchers = getSpecialMatchers(matcher)
135+
const idxReq = matcherIdxRequest(matcher, specialMatchers, fromTimeSec, toTimeSec)
136+
const withIdxReq = new Sql.With('idx', idxReq)
137+
const specialClauses = specialMatchersQuery(specialMatchers.matchers,
138+
'sample_types_units')
139+
const serviceNameSelector = serviceNameSelectorQuery(matcher)
140+
const req = (new Sql.Select()).with(withIdxReq)
141+
.select('val')
142+
.distinct(true)
143+
.from(`profiles_series_gin${dist}`)
144+
.where(Sql.And(
145+
specialClauses,
146+
serviceNameSelector,
147+
Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)),
148+
Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)),
149+
Sql.Eq('key', name),
150+
new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxReq))
151+
))
152+
console.log(req.toString())
153+
promises.push(clickhouse.rawRequest(req.toString() + ' FORMAT JSON', null, DATABASE_NAME()))
154+
}
155+
const labelValues = await Promise.all(promises)
156+
const labelValuesDedup = Object.fromEntries(
157+
labelValues.flatMap(val => val.data.data.map(row => [row.val, true]))
158+
)
91159
const resp = new types.LabelValuesResponse()
92-
resp.setNamesList(labelValues.data.data.map(label => label.val))
160+
resp.setNamesList([...Object.keys(labelValuesDedup)])
93161
return resp
94162
}
95163

@@ -244,6 +312,36 @@ const selectMergeProfile = async (req, res) => {
244312
}
245313
}
246314

315+
/**
316+
*
317+
* @param labelSelector {string}
318+
* @param specialMatchers {object || undefined}
319+
* @param fromTimeSec {number}
320+
* @param toTimeSec {number}
321+
* @returns {Sql.Select}
322+
*/
323+
const matcherIdxRequest = (labelSelector, specialMatchers, fromTimeSec, toTimeSec) => {
324+
specialMatchers = specialMatchers || getSpecialMatchers(labelSelector)
325+
const specialClauses = specialMatchersQuery(specialMatchers.matchers,
326+
'sample_types_units')
327+
const serviceNameSelector = serviceNameSelectorQuery(labelSelector)
328+
const idxReq = (new Sql.Select())
329+
.select(new Sql.Raw('fingerprint'))
330+
.from(`${DATABASE_NAME()}.profiles_series_gin`)
331+
.where(
332+
Sql.And(
333+
specialClauses,
334+
serviceNameSelector,
335+
Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)),
336+
Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`))
337+
)
338+
)
339+
if (!specialMatchers.query.match(/^[{} ]*$/)) {
340+
labelSelectorQuery(idxReq, specialMatchers.query)
341+
}
342+
return idxReq
343+
}
344+
247345
const series = async (req, res) => {
248346
const _req = req.body
249347
const fromTimeSec = Math.floor(_req.getStart && _req.getStart()
@@ -256,19 +354,40 @@ const series = async (req, res) => {
256354
const promises = []
257355
for (const labelSelector of _req.getMatchersList() || []) {
258356
const specialMatchers = getSpecialMatchers(labelSelector)
259-
const specialClauses = specialMatchersQuery(specialMatchers.matchers)
357+
// Special matchers -> query clauses
358+
const sampleTypesUnitsFieldName = '_sample_types_units'
359+
const clauses = []
360+
if (specialMatchers.__name__) {
361+
clauses.push(matcherClause("splitByChar(':', type_id)[1]", specialMatchers.__name__))
362+
}
363+
if (specialMatchers.__period_type__) {
364+
clauses.push(matcherClause("splitByChar(':', type_id)[2]", specialMatchers.__period_type__))
365+
}
366+
if (specialMatchers.__period_unit__) {
367+
clauses.push(matcherClause("splitByChar(':', type_id)[3]", specialMatchers.__period_unit__))
368+
}
369+
if (specialMatchers.__sample_type__) {
370+
clauses.push(matcherClause(`${sampleTypesUnitsFieldName}.1`, specialMatchers.__sample_type__))
371+
}
372+
if (specialMatchers.__sample_unit__) {
373+
clauses.push(matcherClause(`${sampleTypesUnitsFieldName}.2`, specialMatchers.__sample_unit__))
374+
}
375+
if (specialMatchers.__profile_type__) {
376+
clauses.push(matcherClause(
377+
`format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], ${sampleTypesUnitsFieldName}.1, ${sampleTypesUnitsFieldName}.2, _parts[2], _parts[3])`,
378+
specialMatchers.__profile_type__))
379+
}
380+
let specialClauses = null
381+
if (clauses.length === 0) {
382+
specialClauses = Sql.Eq(new Sql.Raw('1'), 1)
383+
} else if (clauses.length === 1) {
384+
specialClauses = clauses[0]
385+
} else {
386+
specialClauses = Sql.And(...clauses)
387+
}
388+
//
260389
const serviceNameSelector = serviceNameSelectorQuery(labelSelector)
261-
const idxReq = (new Sql.Select())
262-
.select(new Sql.Raw('fingerprint'))
263-
.from(`${DATABASE_NAME()}.profiles_series_gin`)
264-
.where(
265-
Sql.And(
266-
serviceNameSelector,
267-
Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)),
268-
Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`))
269-
)
270-
)
271-
labelSelectorQuery(idxReq, specialMatchers.query)
390+
const idxReq = matcherIdxRequest(labelSelector, specialMatchers, fromTimeSec, toTimeSec)
272391
const withIdxReq = (new Sql.With('idx', idxReq, !!clusterName))
273392
const labelsReq = (new Sql.Select())
274393
.with(withIdxReq)
@@ -349,8 +468,21 @@ const series = async (req, res) => {
349468
}
350469

351470
/**
471+
* returns special matchers and sanitized query without them as following:
472+
* @example
473+
* {
474+
* "matchers": {
475+
* "__name__": ["=", "foo"],
476+
* "__period_type__": ["=~", "bar"],
477+
* },
478+
* "query": "{service_name=\"abc\", job=\"def\"}"
479+
* }
352480
*
353481
* @param query {string}
482+
* @returns {{
483+
* matchers: { [fieldName: string]: [operator: string, value: string] },
484+
* query: string
485+
* }}
354486
*/
355487
const getSpecialMatchers = (query) => {
356488
if (query.length <= 2) {
@@ -395,27 +527,45 @@ const matcherClause = (field, matcher) => {
395527
return valRul
396528
}
397529

398-
const specialMatchersQuery = (matchers) => {
530+
/**
531+
* @example
532+
* specialMatchersQuery({
533+
* "__name__": ["=", "foo"],
534+
* "__period_type__": ["=~", "bar"],
535+
* })
536+
*
537+
* @param specialMatchers {Object}
538+
* @returns {Sql.Condition}
539+
*/
540+
const specialMatchersQuery = (specialMatchers) => {
541+
const sampleTypesUnitsFieldName = 'sample_types_units'
399542
const clauses = []
400-
if (matchers.__name__) {
401-
clauses.push(matcherClause("splitByChar(':', type_id)[1]", matchers.__name__))
543+
if (specialMatchers.__name__) {
544+
clauses.push(matcherClause("splitByChar(':', type_id)[1]", specialMatchers.__name__))
402545
}
403-
if (matchers.__period_type__) {
404-
clauses.push(matcherClause("splitByChar(':', type_id)[2]", matchers.__period_type__))
546+
if (specialMatchers.__period_type__) {
547+
clauses.push(matcherClause("splitByChar(':', type_id)[2]", specialMatchers.__period_type__))
405548
}
406-
if (matchers.__period_unit__) {
407-
clauses.push(matcherClause("splitByChar(':', type_id)[3]", matchers.__period_unit__))
549+
if (specialMatchers.__period_unit__) {
550+
clauses.push(matcherClause("splitByChar(':', type_id)[3]", specialMatchers.__period_unit__))
551+
}
552+
const arrayExists = (field) => {
553+
const arrayExists = Sql.Condition(null, null, null)
554+
arrayExists.toString = () => {
555+
return `arrayExists(x -> ${field}, ${sampleTypesUnitsFieldName})`
556+
}
557+
return arrayExists
408558
}
409-
if (matchers.__sample_type__) {
410-
clauses.push(matcherClause('_sample_types_units.1', matchers.__sample_type__))
559+
if (specialMatchers.__sample_type__) {
560+
clauses.push(arrayExists(matcherClause('x.1', specialMatchers.__sample_type__)))
411561
}
412-
if (matchers.__sample_unit__) {
413-
clauses.push(matcherClause('_sample_types_units.2', matchers.__sample_unit__))
562+
if (specialMatchers.__sample_unit__) {
563+
clauses.push(arrayExists(matcherClause('x.2', specialMatchers.__sample_unit__)))
414564
}
415-
if (matchers.__profile_type__) {
416-
clauses.push(matcherClause(
417-
"format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], _sample_types_units.1, _sample_types_units.2, _parts[2], _parts[3])",
418-
matchers.__profile_type__))
565+
if (specialMatchers.__profile_type__) {
566+
clauses.push(arrayExists(matcherClause(
567+
"format('{}:{}:{}:{}:{}', (splitByChar(':', type_id) as _parts)[1], x.1, x.2, _parts[2], _parts[3])",
568+
specialMatchers.__profile_type__)))
419569
}
420570
if (clauses.length === 0) {
421571
return Sql.Eq(new Sql.Raw('1'), 1)

‎pyroscope/render.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const render = async (req, res) => {
1313
? Math.floor(parseInt(req.query.until) / 1000)
1414
: Math.floor((Date.now() - 1000 * 60 * 60 * 48) / 1000)
1515
if (!parsedQuery) {
16-
return res.sendStatus(400).send('Invalid query')
16+
return res.code(400).send('Invalid query')
1717
}
1818
const groupBy = req.query.groupBy || []
1919
let agg = ''
@@ -26,7 +26,7 @@ const render = async (req, res) => {
2626
break
2727
}
2828
if (req.query.format === 'dot') {
29-
return res.sendStatus(400).send('Dot format is not supported')
29+
return res.code(400).send('Dot format is not supported')
3030
}
3131
const promises = []
3232
promises.push(mergeStackTraces(

‎test/e2e

Submodule e2e updated from 376a7db to 7d77675

0 commit comments

Comments
 (0)