Skip to content

Commit 78381e4

Browse files
feat: added more metrics to the ingestion benchmark
feat: added script to transform the report/README output into a docx so that it can be uploaded to gdoc
1 parent 4a1ebcb commit 78381e4

File tree

7 files changed

+149
-16
lines changed

7 files changed

+149
-16
lines changed

benchmarks/ingestion_path_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ var _ = Describe("Ingestion Path", func() {
6666
job := benchCfg.Metrics.Jobs.Distributor
6767
annotation := metrics.DistributorAnnotation
6868

69+
err = metricsClient.MeasureResourceUsageMetrics(e, job, samplingRange, annotation)
70+
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
6971
err = metricsClient.MeasureHTTPRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange, annotation)
7072
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
7173

@@ -79,7 +81,7 @@ var _ = Describe("Ingestion Path", func() {
7981
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
8082
err = metricsClient.MeasureGRPCRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange, annotation)
8183
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
82-
err = metricsClient.MeasureBoltDBShipperRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange)
84+
err = metricsClient.MeasureIndexRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange)
8385
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
8486
}, samplingCfg)
8587
})

benchmarks/query_path_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ var _ = Describe("Query Path", func() {
122122
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
123123
err = metricsClient.MeasureGRPCRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange, annotation)
124124
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
125-
err = metricsClient.MeasureBoltDBShipperRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange)
125+
err = metricsClient.MeasureIndexRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange)
126126
Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
127127
}, samplingCfg)
128128
})

hack/scripts/create-gdoc.py

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import os
2+
import sys
3+
import argparse
4+
from docx import Document
5+
from docx.shared import Inches
6+
from docx.oxml.ns import qn
7+
from docx.oxml import OxmlElement
8+
import markdown
9+
from bs4 import BeautifulSoup
10+
11+
def add_hyperlink(paragraph, url, text, color="0000FF", underline=True):
12+
part = paragraph.part
13+
r_id = part.relate_to(url, 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink', is_external=True)
14+
15+
hyperlink = OxmlElement('w:hyperlink')
16+
hyperlink.set(qn('r:id'), r_id)
17+
18+
new_run = OxmlElement('w:r')
19+
rPr = OxmlElement('w:rPr')
20+
21+
if color:
22+
c = OxmlElement('w:color')
23+
c.set(qn('w:val'), color)
24+
rPr.append(c)
25+
26+
if underline:
27+
u = OxmlElement('w:u')
28+
u.set(qn('w:val'), 'single')
29+
rPr.append(u)
30+
31+
new_run.append(rPr)
32+
new_run.text = text
33+
hyperlink.append(new_run)
34+
35+
paragraph._p.append(hyperlink)
36+
return hyperlink
37+
38+
def add_table_of_contents(soup, doc):
39+
toc = soup.find('ul')
40+
if toc:
41+
for li in toc.find_all('li'):
42+
link = li.find('a')
43+
if link and link['href'].startswith('#'):
44+
heading_text = link.text
45+
toc_paragraph = doc.add_paragraph()
46+
add_hyperlink(toc_paragraph, f'#{heading_text}', heading_text)
47+
48+
def add_markdown_to_docx(md_content, doc, base_path):
49+
html = markdown.markdown(md_content)
50+
soup = BeautifulSoup(html, 'html.parser')
51+
52+
heading_map = {}
53+
toc_inserted = False
54+
55+
for element in soup:
56+
if element.name == 'h1':
57+
paragraph = doc.add_heading(element.text, level=1)
58+
heading_map[element.text] = paragraph
59+
elif element.name == 'h2':
60+
paragraph = doc.add_heading(element.text, level=2)
61+
heading_map[element.text] = paragraph
62+
if element.text.lower() == 'table of contents' and not toc_inserted:
63+
add_table_of_contents(soup, doc)
64+
toc_inserted = True
65+
elif element.name == 'h3':
66+
paragraph = doc.add_heading(element.text, level=3)
67+
heading_map[element.text] = paragraph
68+
elif element.name == 'p':
69+
paragraph = doc.add_paragraph(element.text)
70+
for img in element.find_all('img'):
71+
img_src = img['src'].lstrip('./')
72+
img_path = os.path.join(base_path, img_src)
73+
if os.path.exists(img_path):
74+
doc.add_picture(img_path, width=Inches(5.0))
75+
else:
76+
paragraph.add_run(f"[Image not found: {img_path}]")
77+
elif element.name == 'ul' and not toc_inserted:
78+
for li in element.find_all('li'):
79+
doc.add_paragraph(li.text, style='ListBullet')
80+
elif element.name == 'ol':
81+
for li in element.find_all('li'):
82+
doc.add_paragraph(li.text, style='ListNumber')
83+
elif element.name == 'a':
84+
paragraph = doc.add_paragraph()
85+
add_hyperlink(paragraph, element['href'], element.text)
86+
87+
for heading_text, paragraph in heading_map.items():
88+
bookmark = OxmlElement('w:bookmarkStart')
89+
bookmark.set(qn('w:id'), str(hash(heading_text)))
90+
bookmark.set(qn('w:name'), heading_text)
91+
paragraph._p.insert(0, bookmark)
92+
bookmark_end = OxmlElement('w:bookmarkEnd')
93+
bookmark_end.set(qn('w:id'), str(hash(heading_text)))
94+
paragraph._p.append(bookmark_end)
95+
96+
def convert_readme_to_docx(readme_dir, output_path):
97+
readme_path = os.path.join(readme_dir, 'README.md')
98+
if not os.path.exists(readme_path):
99+
print(f"README.md not found in {readme_dir}")
100+
return
101+
102+
with open(readme_path, 'r') as file:
103+
md_content = file.read()
104+
105+
doc = Document()
106+
add_markdown_to_docx(md_content, doc, readme_dir)
107+
doc.save(output_path)
108+
109+
if __name__ == "__main__":
110+
parser = argparse.ArgumentParser(description='Convert a README.md file to a DOCX file.')
111+
parser.add_argument('readme_dir', type=str, help='Directory containing the README.md file')
112+
args = parser.parse_args()
113+
114+
readme_dir = args.readme_dir
115+
output_path = os.path.join(readme_dir, 'README.docx')
116+
convert_readme_to_docx(readme_dir, output_path)
117+
print(f"Converted README.md in {readme_dir} to {output_path}")

hack/scripts/generate_report.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,13 @@ def plot_measurement(measurements, output_dir, plot_index):
2525
values = measurement['Values']
2626
units = measurement['Units']
2727
annotations = measurement.get('Annotations', [])
28-
component = annotations[0].capitalize() if annotations else 'Unknown Component'
2928

3029
# Generate time values for x-axis starting from 3 minutes
3130
time_values = [(i + 1) * 3 for i in range(len(values))]
3231

3332
plt.plot(time_values, values, marker='o', label=description)
3433

35-
plt.title(f'{component} {name}')
34+
plt.title(f'{name}')
3635
plt.xlabel('Time (minutes)')
3736
plt.ylabel(f'{units}')
3837
plt.legend()
@@ -42,7 +41,7 @@ def plot_measurement(measurements, output_dir, plot_index):
4241
plt.savefig(plot_filename)
4342
plt.close()
4443

45-
return f'./plots/plot_{plot_index}.png', f'{component} {name}'
44+
return f'./plots/plot_{plot_index}.png', f'{name}'
4645

4746
# Collect all measurements from the provided directories
4847
all_measurements = {}

internal/metrics/client.go

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ func (c *Client) MeasureGRPCRequestMetrics(
110110
}
111111
}
112112

113-
func (c *Client) MeasureBoltDBShipperRequestMetrics(
113+
func (c *Client) MeasureIndexRequestMetrics(
114114
e *gmeasure.Experiment,
115115
path RequestPath,
116116
job string,
@@ -249,29 +249,43 @@ func (c *Client) measureCommonRequestMetrics(
249249
sampleRange model.Duration,
250250
annotation gmeasure.Annotation,
251251
) error {
252-
var name, code, requestRateName string
252+
var name, code, badCode, requestRateName, badRequestRateName string
253253

254254
if method == GRPCMethod {
255-
name = fmt.Sprintf("successful GRPC %s", route)
255+
name = fmt.Sprintf("%s successful GRPC %s", job, route)
256256
code = "success"
257-
258257
requestRateName = name
259258
if pathRoutes == GRPCReadPathRoutes {
260259
requestRateName = "successful GRPC reads"
261260
}
261+
262+
badCode = "error|cancel"
263+
badRequestRateName = fmt.Sprintf("%s unsuccessful GRPC %s", job, route)
264+
if pathRoutes == GRPCReadPathRoutes {
265+
requestRateName = "unsuccessful GRPC reads"
266+
}
262267
} else {
263-
name = fmt.Sprintf("2xx %s", route)
268+
name = fmt.Sprintf("%s 2xx %s", job, route)
264269
code = "2.*"
265-
266270
requestRateName = name
267271
if pathRoutes == HTTPReadPathRoutes {
268272
requestRateName = "2xx reads"
269273
}
274+
275+
badCode = "5.*"
276+
badRequestRateName = fmt.Sprintf("%s 5xx %s", job, route)
277+
if pathRoutes == HTTPReadPathRoutes {
278+
badRequestRateName = "5xx reads"
279+
}
270280
}
271281

282+
// Rate request of 200 or success
272283
if err := c.Measure(e, RequestRate(requestRateName, job, pathRoutes, code, sampleRange, annotation)); err != nil {
273284
return err
274285
}
286+
if err := c.Measure(e, RequestRate(badRequestRateName, job, pathRoutes, badCode, sampleRange, annotation)); err != nil {
287+
return err
288+
}
275289
if err := c.Measure(e, RequestDurationAverage(name, job, method, route, code, sampleRange, annotation)); err != nil {
276290
return err
277291
}

internal/metrics/requests.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ func RequestDurationQuantile(
7979
) Measurement {
8080
return Measurement{
8181
Name: fmt.Sprintf("%s request duration P%d", name, percentile),
82+
// clamp_min is used to avoid NaN which breaks the reporting
8283
Query: fmt.Sprintf(
83-
`histogram_quantile(0.%d, sum by (job, le) (rate(loki_request_duration_seconds_bucket{job=~".*%s.*", method="%s", route=~"%s", status_code=~"%s"}[%s]))) * %d`,
84+
`histogram_quantile(0.%d, sum by (job, le) (clamp_min(rate(loki_request_duration_seconds_bucket{job=~".*%s.*", method="%s", route=~"%s", status_code=~"%s"}[%s]), 0.01))) * %d`,
8485
percentile, job, method, route, code, duration, SecondsToMillisecondsMultiplier,
8586
),
8687
Unit: MillisecondsUnit,

internal/metrics/resources.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99

1010
func ContainerCPU(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
1111
return Measurement{
12-
Name: "Sum of Container CPU Usage",
12+
Name: fmt.Sprintf("%s Sum of Container CPU Usage", job),
1313
Query: fmt.Sprintf(
1414
`sum(avg_over_time(pod:container_cpu_usage:sum{pod=~".*%s.*"}[%s])) * %d`,
1515
job, duration, CoresToMillicores,
@@ -21,7 +21,7 @@ func ContainerCPU(job string, duration model.Duration, annotation gmeasure.Annot
2121

2222
func ContainerMemoryWorkingSetBytes(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
2323
return Measurement{
24-
Name: "Sum of Container WorkingSet Memory",
24+
Name: fmt.Sprintf("%s Sum of Container WorkingSet Memory", job),
2525
Query: fmt.Sprintf(
2626
`sum(avg_over_time(container_memory_working_set_bytes{pod=~".*%s.*", container=""}[%s]) / %d)`,
2727
job, duration, BytesToGigabytesMultiplier,
@@ -33,7 +33,7 @@ func ContainerMemoryWorkingSetBytes(job string, duration model.Duration, annotat
3333

3434
func ContainerGoMemstatsHeapInuse(job string, _ model.Duration, annotation gmeasure.Annotation) Measurement {
3535
return Measurement{
36-
Name: "Sum of Container Go Memstats Heap Inuse",
36+
Name: fmt.Sprintf("%s Sum of Container Go Memstats Heap Inuse", job),
3737
Query: fmt.Sprintf(
3838
`sum(go_memstats_heap_inuse_bytes{pod=~".*%s.*"}) / %d`,
3939
job, BytesToGigabytesMultiplier,
@@ -45,7 +45,7 @@ func ContainerGoMemstatsHeapInuse(job string, _ model.Duration, annotation gmeas
4545

4646
func PersistentVolumeUsedBytes(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
4747
return Measurement{
48-
Name: "Sum of Persistent Volume Used Bytes",
48+
Name: fmt.Sprintf("%s Sum of Persistent Volume Used Bytes", job),
4949
Query: fmt.Sprintf(
5050
`sum(avg_over_time(kubelet_volume_stats_used_bytes{persistentvolumeclaim=~".*%s.*"}[%s]) / %d)`,
5151
job, duration, BytesToGigabytesMultiplier,

0 commit comments

Comments
 (0)