Skip to content

Commit ad7d764

Browse files
feat(s3-perf-test): Create s3-perf-test image
This will run a perf test on a given S3 bucket reading a set amount of objects in 16KiB blocks to simulate a real world workload on an object store.
1 parent d6cea4b commit ad7d764

File tree

5 files changed

+372
-0
lines changed

5 files changed

+372
-0
lines changed

.github/workflows/s3-perf-test.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
on:
2+
workflow_dispatch:
3+
inputs:
4+
commit:
5+
description: 'Commit to build'
6+
required: true
7+
default: 'master'
8+
push:
9+
paths:
10+
- "s3-perf-test/**"
11+
- ".github/workflows/s3-perf-test.yml"
12+
- ".github/workflows/build.yml"
13+
14+
15+
jobs:
16+
build:
17+
uses: ./.github/workflows/build.yml
18+
secrets: inherit
19+
with:
20+
image-name: s3-perf-test
21+
folder: s3-perf-test
22+
build-args: "--build-arg COMMIT=${{ github.event.inputs.commit }}"

s3-perf-test/Dockerfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM golang:1.23.2-bookworm AS builder
2+
ARG TARGETOS
3+
ARG TARGETARCH
4+
5+
WORKDIR /app
6+
7+
COPY . .
8+
9+
RUN go build -o /s3-perf-test main.go
10+
11+
FROM debian:bookworm-slim
12+
13+
RUN apt-get update && apt-get install -y ca-certificates
14+
15+
RUN update-ca-certificates
16+
17+
COPY --from=builder /s3-perf-test .
18+
19+
ENTRYPOINT [ "/s3-perf-test" ]

s3-perf-test/go.mod

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
module github.com/coreweave/ml-containers/s3-perf-test
2+
3+
go 1.23.2
4+
5+
require (
6+
github.com/hashicorp/go-cleanhttp v0.5.2
7+
github.com/jedib0t/go-pretty/v6 v6.6.7
8+
github.com/minio/minio-go/v7 v7.0.87
9+
)
10+
11+
require (
12+
github.com/dustin/go-humanize v1.0.1 // indirect
13+
github.com/go-ini/ini v1.67.0 // indirect
14+
github.com/goccy/go-json v0.10.5 // indirect
15+
github.com/google/uuid v1.6.0 // indirect
16+
github.com/klauspost/compress v1.17.11 // indirect
17+
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
18+
github.com/mattn/go-runewidth v0.0.16 // indirect
19+
github.com/minio/crc64nvme v1.0.1 // indirect
20+
github.com/minio/md5-simd v1.1.2 // indirect
21+
github.com/rivo/uniseg v0.4.7 // indirect
22+
github.com/rs/xid v1.6.0 // indirect
23+
golang.org/x/crypto v0.33.0 // indirect
24+
golang.org/x/net v0.35.0 // indirect
25+
golang.org/x/sys v0.30.0 // indirect
26+
golang.org/x/text v0.22.0 // indirect
27+
)

s3-perf-test/go.sum

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
4+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
5+
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
6+
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
7+
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
8+
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
9+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
10+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
11+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
12+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
13+
github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPItz/Uuo=
14+
github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU=
15+
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
16+
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
17+
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
18+
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
19+
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
20+
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
21+
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
22+
github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
23+
github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
24+
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
25+
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
26+
github.com/minio/minio-go/v7 v7.0.87 h1:nkr9x0u53PespfxfUqxP3UYWiE2a41gaofgNnC4Y8WQ=
27+
github.com/minio/minio-go/v7 v7.0.87/go.mod h1:33+O8h0tO7pCeCWwBVa07RhVVfB/3vS4kEX7rwYKmIg=
28+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
29+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
30+
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
31+
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
32+
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
33+
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
34+
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
35+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
36+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
37+
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
38+
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
39+
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
40+
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
41+
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
42+
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
43+
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
44+
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
45+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
46+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

s3-perf-test/main.go

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"log/slog"
8+
"math/rand"
9+
"os"
10+
"strconv"
11+
"sync"
12+
"time"
13+
14+
"github.com/hashicorp/go-cleanhttp"
15+
"github.com/jedib0t/go-pretty/v6/table"
16+
"github.com/minio/minio-go/v7"
17+
"github.com/minio/minio-go/v7/pkg/credentials"
18+
)
19+
20+
func main() {
21+
accessKeyID := os.Getenv("S3_ACCESS_KEY_ID")
22+
secretKey := os.Getenv("S3_ACCESS_KEY_SECRET_KEY")
23+
region := os.Getenv("S3_REGION")
24+
endpoint := os.Getenv("S3_ENDPOINT")
25+
gomaxprocs := os.Getenv("GOMAXPROCS")
26+
bucket := os.Getenv("S3_BUCKET")
27+
prefix := os.Getenv("S3_PREFIX")
28+
objectCount := os.Getenv("MAX_OBJECTS")
29+
testDuration := os.Getenv("TEST_DURATION")
30+
useSSL := os.Getenv("USE_SSL")
31+
32+
if accessKeyID == "" || secretKey == "" {
33+
slog.Error("S3_ACCESS_KEY_ID and S3_ACCESS_KEY_SECRET_KEY are required to run")
34+
35+
os.Exit(1)
36+
}
37+
38+
if region == "" || endpoint == "" {
39+
slog.Error("S3_REGION and S3_ENDPOINT are required to run")
40+
41+
os.Exit(1)
42+
}
43+
44+
if bucket == "" {
45+
slog.Error("S3_BUCKET is required to run")
46+
47+
os.Exit(1)
48+
}
49+
50+
threads := 1
51+
maxObjects := 1000
52+
var err error
53+
54+
if gomaxprocs != "" {
55+
threads, err = strconv.Atoi(gomaxprocs)
56+
if err != nil {
57+
slog.Error("failed to convert GOMAXPROCS to int", "error", err)
58+
59+
os.Exit(1)
60+
}
61+
}
62+
63+
if objectCount != "" {
64+
maxObjects, err = strconv.Atoi(objectCount)
65+
if err != nil {
66+
slog.Error("failed to convert MAX_OBJECTS to int", "error", err)
67+
68+
os.Exit(1)
69+
}
70+
}
71+
72+
duration := time.Minute * 10
73+
74+
if testDuration != "" {
75+
duration, err = time.ParseDuration(testDuration)
76+
if err != nil {
77+
slog.Error("failed to parse TEST_DURATION", "error", err)
78+
79+
os.Exit(1)
80+
}
81+
}
82+
83+
ssl := true
84+
if useSSL != "" {
85+
ssl, err = strconv.ParseBool(useSSL)
86+
if err != nil {
87+
slog.Error("failed to parse USE_SSL", "error", err)
88+
89+
os.Exit(1)
90+
}
91+
}
92+
93+
ctx, cancel := context.WithCancel(context.Background())
94+
transport := cleanhttp.DefaultPooledTransport()
95+
96+
s3Client, err := minio.New(endpoint, &minio.Options{
97+
Creds: credentials.NewStaticV4(accessKeyID, secretKey, ""),
98+
Secure: ssl,
99+
Transport: transport,
100+
Region: region,
101+
BucketLookup: minio.BucketLookupDNS,
102+
})
103+
if err != nil {
104+
slog.Error("failed to create s3 client", "error", err)
105+
106+
os.Exit(1)
107+
}
108+
109+
var objects []string
110+
var sizes []int64
111+
112+
objChan := s3Client.ListObjects(ctx, bucket, minio.ListObjectsOptions{
113+
WithMetadata: true,
114+
Prefix: prefix,
115+
MaxKeys: maxObjects,
116+
})
117+
118+
for obj := range objChan {
119+
if obj.Err != nil {
120+
slog.Error("error listing objects", "error", err)
121+
122+
os.Exit(1)
123+
}
124+
125+
objects = append(objects, obj.Key)
126+
sizes = append(sizes, obj.Size)
127+
}
128+
129+
rng := rand.New(rand.NewSource(time.Now().Unix()))
130+
timer := time.NewTimer(duration)
131+
resultChan := make(chan *testResult, threads)
132+
wg := &sync.WaitGroup{}
133+
134+
params := &testParams{
135+
Bucket: bucket,
136+
RNG: rng,
137+
Objects: objects,
138+
ObjectSizes: sizes,
139+
S3Client: s3Client,
140+
}
141+
142+
start := time.Now().UTC()
143+
144+
for i := range threads {
145+
wg.Add(1)
146+
go func() {
147+
result := runTest(ctx, params, i)
148+
resultChan <- result
149+
wg.Done()
150+
}()
151+
}
152+
153+
<-timer.C
154+
155+
slog.Info("test done")
156+
157+
// Tell our tests that we're done and wait
158+
cancel()
159+
wg.Wait()
160+
161+
aggregatedBytesRead := 0
162+
aggregatedRequestsSent := 0
163+
var averageTTLB int64
164+
165+
for range threads {
166+
result := <-resultChan
167+
168+
aggregatedBytesRead += int(result.TotalBytesRead)
169+
aggregatedRequestsSent += int(result.TotalRequestsSent)
170+
averageTTLB += result.TotalTTLBMS
171+
}
172+
173+
averageTTLB = averageTTLB / int64(aggregatedRequestsSent)
174+
175+
timeSpent := time.Since(start)
176+
177+
t := table.NewWriter()
178+
t.SetOutputMirror(os.Stdout)
179+
t.AppendHeader(table.Row{"", "RESULTS"})
180+
t.AppendRow(table.Row{"Time Spent", timeSpent.String()})
181+
t.AppendRow(table.Row{"Total Bytes Read", fmt.Sprintf("%d", aggregatedBytesRead)})
182+
t.AppendRow(table.Row{"Total Requests Sent", fmt.Sprintf("%d", aggregatedRequestsSent)})
183+
t.AppendRow(table.Row{"Average TTLB", fmt.Sprintf("%d", averageTTLB)})
184+
185+
t.Render()
186+
}
187+
188+
type testParams struct {
189+
Bucket string
190+
RNG *rand.Rand
191+
Objects []string
192+
ObjectSizes []int64
193+
S3Client *minio.Client
194+
}
195+
196+
type testResult struct {
197+
TotalBytesRead int64
198+
TotalRequestsSent int64
199+
TotalTTLBMS int64
200+
}
201+
202+
func runTest(ctx context.Context, params *testParams, id int) *testResult {
203+
ll := slog.With("ID", id)
204+
205+
ll.Info("starting test")
206+
207+
result := &testResult{}
208+
209+
testCtx := context.Background()
210+
211+
for {
212+
select {
213+
case <-ctx.Done():
214+
ll.Info("context cancelled, stopping test")
215+
216+
return result
217+
default:
218+
// Get our random object
219+
randObjIndex := params.RNG.Int() % len(params.Objects)
220+
obj := params.Objects[randObjIndex]
221+
size := params.ObjectSizes[randObjIndex]
222+
223+
// Get a random 16KiB offset to read
224+
maxOffset := size / (16 * 1024)
225+
randObjOffset := params.RNG.Int() % int(maxOffset)
226+
227+
rangeStart := int64(randObjOffset * (16 * 1024))
228+
rangeEnd := min(int64((rangeStart + (16 * 1024))), size)
229+
230+
result.TotalRequestsSent++
231+
232+
start := time.Now().UTC()
233+
234+
reqOpts := minio.GetObjectOptions{}
235+
reqOpts.SetRange(rangeStart, rangeEnd)
236+
237+
resp, err := params.S3Client.GetObject(testCtx, params.Bucket, obj, reqOpts)
238+
239+
if err != nil {
240+
ll.Error("failed to fetch range", "error", err)
241+
242+
continue
243+
}
244+
245+
amount, err := io.Copy(io.Discard, resp)
246+
resp.Close()
247+
248+
result.TotalBytesRead += amount
249+
result.TotalTTLBMS += time.Since(start).Milliseconds()
250+
251+
if err != nil {
252+
ll.Error("failed to discard response body", "error", err)
253+
254+
continue
255+
}
256+
}
257+
}
258+
}

0 commit comments

Comments
 (0)