Skip to content

Commit e2abbc3

Browse files
jaeoptjaeopt
authored
fix: support arbitrarily long list of user attributes in batch events (#386)
- WorkManager has 10KB limit on Data, so we cannot support batch events for a long list of attributes or large batch sizes. - BatchEvents are compressed/decompressed to support large events. - Compression/decompression is bypassed when the data size is less than the 10KB limit.
1 parent 8dcf1b7 commit e2abbc3

File tree

5 files changed

+424
-14
lines changed

5 files changed

+424
-14
lines changed

‎event-handler/build.gradle

+3
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,14 @@ dependencies {
5151
api project(':shared')
5252
implementation "androidx.annotation:annotation:$annotations_ver"
5353
implementation "androidx.work:work-runtime:$work_runtime"
54+
// Base64
55+
implementation "commons-codec:commons-codec:1.15"
5456

5557
compileOnly "com.noveogroup.android:android-logger:$android_logger_ver"
5658

5759
testImplementation "junit:junit:$junit_ver"
5860
testImplementation "org.mockito:mockito-core:$mockito_ver"
61+
testImplementation "org.powermock:powermock-mockito-release-full:$powermock_ver"
5962
testImplementation "com.noveogroup.android:android-logger:$android_logger_ver"
6063

6164
androidTestImplementation "androidx.test.ext:junit:$androidx_test"

‎event-handler/src/main/java/com/optimizely/ab/android/event_handler/EventHandlerUtils.java

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package com.optimizely.ab.android.event_handler;
2+
3+
import static java.util.zip.Deflater.BEST_COMPRESSION;
4+
5+
import android.os.Build;
6+
7+
import androidx.annotation.NonNull;
8+
import androidx.annotation.RequiresApi;
9+
import androidx.annotation.VisibleForTesting;
10+
11+
import org.apache.commons.codec.binary.Base64;
12+
13+
import java.io.ByteArrayInputStream;
14+
import java.io.ByteArrayOutputStream;
15+
import java.io.IOException;
16+
import java.nio.charset.Charset;
17+
import java.nio.charset.StandardCharsets;
18+
import java.util.zip.Deflater;
19+
import java.util.zip.GZIPInputStream;
20+
import java.util.zip.GZIPOutputStream;
21+
import java.util.zip.Inflater;
22+
23+
public class EventHandlerUtils {
24+
25+
private static final int BUFFER_SIZE = 32*1024;
26+
27+
public static String compress(@NonNull String decompressed) throws IOException {
28+
byte[] data = decompressed.getBytes();
29+
30+
final Deflater deflater = new Deflater();
31+
deflater.setInput(data);
32+
33+
try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(data.length)) {
34+
deflater.finish();
35+
final byte[] buffer = new byte[BUFFER_SIZE];
36+
while (!deflater.finished()) {
37+
final int count = deflater.deflate(buffer);
38+
outputStream.write(buffer, 0, count);
39+
}
40+
41+
byte[] bytes = outputStream.toByteArray();
42+
// encoded to Base64 (instead of byte[] since WorkManager.Data size is unexpectedly expanded with byte[]).
43+
return encodeToBase64(bytes);
44+
}
45+
}
46+
47+
public static String decompress(@NonNull String base64) throws Exception {
48+
byte[] data = decodeFromBase64(base64);
49+
50+
final Inflater inflater = new Inflater();
51+
inflater.setInput(data);
52+
53+
try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(data.length)) {
54+
byte[] buffer = new byte[BUFFER_SIZE];
55+
while (!inflater.finished()) {
56+
final int count = inflater.inflate(buffer);
57+
outputStream.write(buffer, 0, count);
58+
}
59+
60+
return outputStream.toString();
61+
}
62+
}
63+
64+
static String encodeToBase64(byte[] bytes) {
65+
// - org.apache.commons.Base64 is used (instead of android.util.Base64) for unit testing
66+
// - encodeBase64() for backward compatibility (instead of encodeBase64String()).
67+
String base64 = "";
68+
if (bytes != null) {
69+
byte[] encoded = Base64.encodeBase64(bytes);
70+
base64 = new String(encoded);
71+
}
72+
return base64;
73+
}
74+
75+
static byte[] decodeFromBase64(String base64) {
76+
return Base64.decodeBase64(base64.getBytes());
77+
}
78+
79+
}

‎event-handler/src/main/java/com/optimizely/ab/android/event_handler/EventWorker.java

+75-14
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import android.content.Context;
1919

2020
import androidx.annotation.NonNull;
21+
import androidx.annotation.Nullable;
22+
import androidx.annotation.VisibleForTesting;
2123
import androidx.work.Data;
2224
import androidx.work.Worker;
2325
import androidx.work.WorkerParameters;
@@ -32,7 +34,8 @@
3234
public class EventWorker extends Worker {
3335
public static final String workerId = "EventWorker";
3436

35-
EventDispatcher eventDispatcher;
37+
@VisibleForTesting
38+
public EventDispatcher eventDispatcher;
3639

3740
public EventWorker(@NonNull Context context, @NonNull WorkerParameters workerParams) {
3841
super(context, workerParams);
@@ -46,30 +49,88 @@ public EventWorker(@NonNull Context context, @NonNull WorkerParameters workerPar
4649
new ServiceScheduler.PendingIntentFactory(context),
4750
LoggerFactory.getLogger(ServiceScheduler.class));
4851
eventDispatcher = new EventDispatcher(context, optlyStorage, eventDAO, eventClient, serviceScheduler, LoggerFactory.getLogger(EventDispatcher.class));
49-
50-
}
51-
52-
public static Data getData(LogEvent event) {
53-
return new Data.Builder()
54-
.putString("url", event.getEndpointUrl())
55-
.putString("body", event.getBody())
56-
.build();
5752
}
5853

5954
@NonNull
6055
@Override
6156
public Result doWork() {
62-
String url = getInputData().getString("url");
63-
String body = getInputData().getString("body");
57+
Data inputData = getInputData();
58+
String url = inputData.getString("url");
59+
String body = getEventBodyFromInputData(inputData);
6460
boolean dispatched = true;
6561

66-
if (url != null && !url.isEmpty() && body != null && !body.isEmpty()) {
62+
if (isEventValid(url, body)) {
6763
dispatched = eventDispatcher.dispatch(url, body);
68-
}
69-
else {
64+
} else {
7065
dispatched = eventDispatcher.dispatch();
7166
}
7267

7368
return dispatched ? Result.success() : Result.retry();
7469
}
70+
71+
public static Data getData(LogEvent event) {
72+
String url = event.getEndpointUrl();
73+
String body = event.getBody();
74+
75+
// androidx.work.Data throws IllegalStateException if total data length is more than MAX_DATA_BYTES
76+
// compress larger body and decompress it before dispatching. The compress rate is very high because of repeated data (20KB -> 1KB, 45KB -> 1.5KB).
77+
78+
int maxSizeBeforeCompress = Data.MAX_DATA_BYTES - 1000; // 1000 reserved for other meta data
79+
80+
if (body.length() < maxSizeBeforeCompress) {
81+
return dataForEvent(url, body);
82+
} else {
83+
return compressEvent(url, body);
84+
}
85+
}
86+
87+
@VisibleForTesting
88+
public static Data compressEvent(String url, String body) {
89+
try {
90+
String compressed = EventHandlerUtils.compress(body);
91+
return dataForCompressedEvent(url, compressed);
92+
} catch (Exception e) {
93+
return dataForEvent(url, body);
94+
}
95+
}
96+
97+
@VisibleForTesting
98+
public static Data dataForEvent(String url, String body) {
99+
return new Data.Builder()
100+
.putString("url", url)
101+
.putString("body", body)
102+
.build();
103+
}
104+
105+
@VisibleForTesting
106+
public static Data dataForCompressedEvent(String url, String compressed) {
107+
return new Data.Builder()
108+
.putString("url", url)
109+
.putString("bodyCompressed", compressed)
110+
.build();
111+
}
112+
113+
@VisibleForTesting
114+
@Nullable
115+
public String getEventBodyFromInputData(Data inputData) {
116+
// check non-compressed data first
117+
118+
String body = inputData.getString("body");
119+
if (body != null) return body;
120+
121+
// check if data compressed
122+
123+
String compressed = inputData.getString("bodyCompressed");
124+
try {
125+
return EventHandlerUtils.decompress(compressed);
126+
} catch (Exception e) {
127+
return null;
128+
}
129+
}
130+
131+
@VisibleForTesting
132+
public boolean isEventValid(String url, String body) {
133+
return url != null && !url.isEmpty() && body != null && !body.isEmpty();
134+
}
135+
75136
}

‎event-handler/src/test/java/com/optimizely/ab/android/event_handler/EventHandlerUtilsTest.java

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package com.optimizely.ab.android.event_handler;
2+
3+
import static org.junit.Assert.assertEquals;
4+
5+
import androidx.work.Data;
6+
7+
import org.junit.Test;
8+
9+
import java.io.IOException;
10+
11+
public class EventHandlerUtilsTest {
12+
13+
@Test
14+
public void compressAndDecompress() throws Exception {
15+
String str = makeRandomString(1000);
16+
17+
String compressed = EventHandlerUtils.compress(str);
18+
assert(compressed.length() < (str.length() * 0.5));
19+
20+
String decompressed = EventHandlerUtils.decompress(compressed);
21+
assertEquals(str, decompressed);
22+
}
23+
24+
@Test(timeout=30000)
25+
public void measureCompressionDelay() throws Exception {
26+
int maxEventSize = 100000; // 100KB (~100 attributes)
27+
int count = 3000;
28+
29+
String body = EventHandlerUtilsTest.makeRandomString(maxEventSize);
30+
31+
long start = System.currentTimeMillis();
32+
for (int i = 0; i < count; i++) {
33+
EventHandlerUtils.compress(body);
34+
}
35+
long end = System.currentTimeMillis();
36+
float delayCompress = ((float)(end - start))/count;
37+
System.out.println("Compression Delay: " + String.valueOf(delayCompress) + " millisecs");
38+
assert(delayCompress < 10); // less than 1ms for 100KB (set 10ms upperbound)
39+
40+
start = System.currentTimeMillis();
41+
for (int i = 0; i < count; i++) {
42+
String compressed = EventHandlerUtils.compress(body);
43+
EventHandlerUtils.decompress(compressed);
44+
}
45+
end = System.currentTimeMillis();
46+
float delayDecompress = ((float)(end - start))/count - delayCompress;
47+
System.out.println("Decompression Delay: " + String.valueOf(delayDecompress) + " millisecs");
48+
assert(delayDecompress < 10); // less than 1ms for 100KB (set 10ms upperbound)
49+
}
50+
51+
public static String makeRandomString(int maxSize) {
52+
StringBuilder builder = new StringBuilder();
53+
54+
// for high compression rate, shift repeated string window.
55+
int window = 100;
56+
int shift = 3; // adjust (1...10) this for compression rate. smaller for higher rates.
57+
58+
int start = 0;
59+
int end = start + window;
60+
int i = 0;
61+
62+
int size = 0;
63+
while (true) {
64+
String str = String.valueOf(i);
65+
size += str.length();
66+
if (size > maxSize) {
67+
break;
68+
}
69+
builder.append(str);
70+
71+
i++;
72+
if (i > end) {
73+
start = start + shift;
74+
end = start + window;
75+
i = start;
76+
}
77+
}
78+
79+
return builder.toString();
80+
}
81+
82+
}

0 commit comments

Comments
 (0)