-
Notifications
You must be signed in to change notification settings - Fork 244
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
New max sustainable rate implementation (#329)
## Motivation Currently the WorkloadGenerator.findMaximumSustainableRate algorithm introduces publish delay. It has been observed the algorithm settles on a publish rate that neither producers nor consumers can keep up with. For example, on a Kafka workload with 100 topics, 1kb message, the algorithm settled on a publish rate of 2.5m msg/s when the producers could only actually achieve 2m msg/s. ## Changes Implement a new algorithm that checks for both receive backlog and publish backlog and adjusts the publish according and has a progressive rate ramp up when there is no observed backlog.
- Loading branch information
Dave Maughan
authored
Oct 24, 2022
1 parent
5967b3f
commit 00502be
Showing
9 changed files
with
325 additions
and
100 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
96 changes: 96 additions & 0 deletions
96
benchmark-framework/src/main/java/io/openmessaging/benchmark/RateController.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.openmessaging.benchmark; | ||
|
|
||
| import static java.util.concurrent.TimeUnit.SECONDS; | ||
| import static lombok.AccessLevel.PACKAGE; | ||
|
|
||
| import io.openmessaging.benchmark.utils.Env; | ||
| import lombok.Getter; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Slf4j | ||
| class RateController { | ||
| private static final long ONE_SECOND_IN_NANOS = SECONDS.toNanos(1); | ||
| private final long publishBacklogLimit; | ||
| private final long receiveBacklogLimit; | ||
| private final double minRampingFactor; | ||
| private final double maxRampingFactor; | ||
|
|
||
| @Getter(PACKAGE) | ||
| private double rampingFactor; | ||
|
|
||
| private long previousTotalPublished = 0; | ||
| private long previousTotalReceived = 0; | ||
|
|
||
| RateController() { | ||
| publishBacklogLimit = Env.getLong("PUBLISH_BACKLOG_LIMIT", 1_000); | ||
| receiveBacklogLimit = Env.getLong("RECEIVE_BACKLOG_LIMIT", 1_000); | ||
| minRampingFactor = Env.getDouble("MIN_RAMPING_FACTOR", 0.01); | ||
| maxRampingFactor = Env.getDouble("MAX_RAMPING_FACTOR", 1); | ||
| rampingFactor = maxRampingFactor; | ||
| } | ||
|
|
||
| double nextRate(double rate, long periodNanos, long totalPublished, long totalReceived) { | ||
| long expected = (long) ((rate / ONE_SECOND_IN_NANOS) * periodNanos); | ||
| long published = totalPublished - previousTotalPublished; | ||
| long received = totalReceived - previousTotalReceived; | ||
|
|
||
| previousTotalPublished = totalPublished; | ||
| previousTotalReceived = totalReceived; | ||
|
|
||
| if (log.isDebugEnabled()) { | ||
| log.debug( | ||
| "Current rate: {} -- Publish rate {} -- Receive Rate: {}", | ||
| rate, | ||
| rate(published, periodNanos), | ||
| rate(received, periodNanos)); | ||
| } | ||
|
|
||
| long receiveBacklog = totalPublished - totalReceived; | ||
| if (receiveBacklog > receiveBacklogLimit) { | ||
| return nextRate(periodNanos, received, expected, receiveBacklog, "Receive"); | ||
| } | ||
|
|
||
| long publishBacklog = expected - published; | ||
| if (publishBacklog > publishBacklogLimit) { | ||
| return nextRate(periodNanos, published, expected, publishBacklog, "Publish"); | ||
| } | ||
|
|
||
| rampUp(); | ||
|
|
||
| return rate + (rate * rampingFactor); | ||
| } | ||
|
|
||
| private double nextRate(long periodNanos, long actual, long expected, long backlog, String type) { | ||
| log.debug("{} backlog: {}", type, backlog); | ||
| rampDown(); | ||
| long nextExpected = Math.max(0, expected - backlog); | ||
| double nextExpectedRate = rate(nextExpected, periodNanos); | ||
| double actualRate = rate(actual, periodNanos); | ||
| return Math.min(actualRate, nextExpectedRate); | ||
| } | ||
|
|
||
| private double rate(long count, long periodNanos) { | ||
| return (count / (double) periodNanos) * ONE_SECOND_IN_NANOS; | ||
| } | ||
|
|
||
| private void rampUp() { | ||
| rampingFactor = Math.min(maxRampingFactor, rampingFactor * 2); | ||
| } | ||
|
|
||
| private void rampDown() { | ||
| rampingFactor = Math.max(minRampingFactor, rampingFactor / 2); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.openmessaging.benchmark.utils; | ||
|
|
||
|
|
||
| import java.util.Optional; | ||
| import java.util.function.Function; | ||
|
|
||
| public final class Env { | ||
| private Env() {} | ||
|
|
||
| public static long getLong(String key, long defaultValue) { | ||
| return get(key, Long::parseLong, defaultValue); | ||
| } | ||
|
|
||
| public static double getDouble(String key, double defaultValue) { | ||
| return get(key, Double::parseDouble, defaultValue); | ||
| } | ||
|
|
||
| public static <T> T get(String key, Function<String, T> function, T defaultValue) { | ||
| return Optional.ofNullable(System.getenv(key)).map(function).orElse(defaultValue); | ||
| } | ||
| } |
70 changes: 70 additions & 0 deletions
70
benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.openmessaging.benchmark; | ||
|
|
||
| import static java.util.concurrent.TimeUnit.SECONDS; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class RateControllerTest { | ||
| private final RateController rateController = new RateController(); | ||
| private double rate = 10_000; | ||
| private long periodNanos = SECONDS.toNanos(1); | ||
|
|
||
| @Test | ||
| void receiveBacklog() { | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
|
|
||
| // no backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000); | ||
| assertThat(rate).isEqualTo(20_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
|
|
||
| // receive backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 20_000, 15_000); | ||
| assertThat(rate).isEqualTo(5_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(0.5); | ||
| } | ||
|
|
||
| @Test | ||
| void publishBacklog() { | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
|
|
||
| // no backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000); | ||
| assertThat(rate).isEqualTo(20_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
|
|
||
| // publish backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 15_000, 20_000); | ||
| assertThat(rate).isEqualTo(5_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(0.5); | ||
| } | ||
|
|
||
| @Test | ||
| void rampUp() { | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
|
|
||
| // receive backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 10_000, 5_000); | ||
| assertThat(rate).isEqualTo(5_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(0.5); | ||
|
|
||
| // no backlog | ||
| rate = rateController.nextRate(rate, periodNanos, 20_000, 20_000); | ||
| assertThat(rate).isEqualTo(10_000); | ||
| assertThat(rateController.getRampingFactor()).isEqualTo(1); | ||
| } | ||
| } |
Oops, something went wrong.