Increase min read size

Author: salvacorts

docs/sources/shared/configuration.md

@@ -4316,6 +4316,16 @@ discover_generic_fields:
 # CLI flag: -querier.query-timeout
 [query_timeout: <duration> | default = 1m]
 
+# Maximum bytes per second for bucket GetObject operations during a query. 0
+# means unlimited. Also expressible in human readable forms (1MB, 256KB, etc).
+# CLI flag: -querier.query-bucket-get-object-rate-limit
+[query_bucket_get_object_rate_limit: <int> | default = 0B]
+
+# Maximum burst bytes for bucket GetObject operations during a query. 0 means
+# unlimited. Also expressible in human readable forms (1MB, 256KB, etc).
+# CLI flag: -querier.query-bucket-get-object-rate-limit-burst
+[query_bucket_get_object_rate_limit_burst: <int> | default = 0B]
+
 # Split queries by a time interval and execute in parallel. The value 0 disables
 # splitting by time. This also determines how cache keys are chosen when result
 # caching is enabled.

pkg/querier/testutil/limits.go

@@ -99,3 +99,11 @@ func (m *MockLimits) DebugEngineStreams(_ string) bool {
 func (m *MockLimits) DebugEngineTasks(_ string) bool {
 	return false
 }
+
+func (m *MockLimits) QueryBucketGetObjectRateLimit(_ context.Context, _ string) int64 {
+	return 0 // 0 means unlimited
+}
+
+func (m *MockLimits) QueryBucketGetObjectRateLimitBurst(_ context.Context, _ string) int64 {
+	return 0 // 0 means unlimited
+}

pkg/storage/bucket/rate_limited_reader.go

@@ -16,7 +16,10 @@ import (
 
 // minReadSize is the minimum chunk size for reading data.
 // This ensures we read in reasonable-sized batches rather than very small ones.
-const minReadSize = 512
+// For typical 1-2MB objects (most of our chunks), 64KB provides a good balance between efficiency
+// and rate limiting responsiveness.
+// E.g. for 2MB object, 64KB read size is 32 reads, which is reasonable.
+const minReadSize = 64 * 1024 // 64KB
 
 type rateLimiterKey struct{}