Skip to content

Commit 0bac600

Browse files
committed
added tests for serialized packed
1 parent 06396df commit 0bac600

File tree

2 files changed

+183
-6
lines changed

2 files changed

+183
-6
lines changed

src/java/org/apache/cassandra/service/accord/serializers/SerializePacked.java

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,17 @@
3131
* A set of simple utilities to quickly serialize/deserialize arrays/lists of values that each require <= 64 bits to represent.
3232
* These are packed into an "array" of fixed bit width, so that the total size consumed is ceil((bits*elements)/8).
3333
* This can (in future) be read directly without deserialization, by indexing into the byte stream directly.
34+
* <p/>
35+
* The serialized value is optimized for values in the range 0 to 256 (negative will be rejected), and should produce
36+
* output smaller or equal to vint serialization; when values are larger than 256, then the packing can produce 1 extra
37+
* serialized byte. Serialization is safe in these cases, and faster to skip.
3438
*/
3539
public class SerializePacked
3640
{
3741
public static void serializePackedSortedIntsAndLength(int[] vs, DataOutputPlus out) throws IOException
3842
{
3943
out.writeUnsignedVInt32(vs.length);
40-
serializePackedSortedInts(vs, out);;
44+
serializePackedSortedInts(vs, out);
4145
}
4246

4347
public static void serializePackedSortedInts(int[] vs, DataOutputPlus out) throws IOException
@@ -46,8 +50,10 @@ public static void serializePackedSortedInts(int[] vs, DataOutputPlus out) throw
4650
return;
4751

4852
int last = vs[vs.length - 1];
53+
Invariants.require(last >= 0,
54+
() -> String.format("Found a negative value at offset %d; value %d", (Object) (vs.length - 1), (Object) last));
4955
out.writeUnsignedVInt32(last);
50-
serializePackedInts(vs, 0, vs.length - 1, last - 1, out);
56+
serializePackedInts(vs, 0, vs.length - 1, last, out);
5157
}
5258

5359
public static int[] deserializePackedSortedIntsAndLength(DataInputPlus in) throws IOException
@@ -62,7 +68,7 @@ public static int[] deserializePackedSortedInts(int length, DataInputPlus in) th
6268

6369
int last = in.readUnsignedVInt32();
6470
int[] vs = new int[length];
65-
deserializePackedInts(vs, 0, length - 1, last - 1, in);
71+
deserializePackedInts(vs, 0, length - 1, last, in);
6672
vs[length - 1] = last;
6773
return vs;
6874
}
@@ -77,7 +83,7 @@ public static void skipPackedSortedInts(int length, DataInputPlus in) throws IOE
7783
if (length > 0)
7884
{
7985
int last = in.readUnsignedVInt32();
80-
skipPackedInts(0, length - 1, last - 1, in);
86+
skipPackedInts(0, length - 1, last, in);
8187
}
8288
}
8389

@@ -91,7 +97,7 @@ public static long serializedSizeOfPackedSortedInts(int[] vs)
9197
if (vs.length == 0)
9298
return 0;
9399
int last = vs[vs.length - 1];
94-
return TypeSizes.sizeofUnsignedVInt(last) + serializedPackedSize(vs.length - 1, last - 1);
100+
return TypeSizes.sizeofUnsignedVInt(last) + serializedPackedSize(vs.length - 1, last);
95101
}
96102

97103
public static void serializePackedInts(int[] vs, int from, int to, long max, DataOutputPlus out) throws IOException
@@ -132,7 +138,9 @@ public static <In> void serializePacked(SerializeAdapter<In> adapter, In in, int
132138
for (int i = from; i < to; i++)
133139
{
134140
long v = adapter.get(in, i);
135-
Invariants.require((v & outOfRange) == 0);
141+
int finalI = i;
142+
Invariants.require(v >= 0 && (v & outOfRange) == 0,
143+
() -> String.format(v < 0 ? "Found a negative value at offset %d; value %d" : "Value out of range at offset %d; value %d", (Object) finalI, (Object) v));
136144
buffer |= v << bufferCount;
137145
bufferCount = bufferCount + bitsPerEntry;
138146
if (bufferCount >= 64)
Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.cassandra.service.accord.serializers;
20+
21+
import java.io.IOException;
22+
import java.util.Arrays;
23+
24+
import accord.utils.Gens;
25+
import org.junit.Test;
26+
27+
import accord.utils.Gen;
28+
import org.apache.cassandra.db.TypeSizes;
29+
import org.apache.cassandra.io.Serializers;
30+
import org.apache.cassandra.io.UnversionedSerializer;
31+
import org.apache.cassandra.io.util.DataInputPlus;
32+
import org.apache.cassandra.io.util.DataOutputBuffer;
33+
import org.apache.cassandra.io.util.DataOutputPlus;
34+
import org.assertj.core.api.Assertions;
35+
36+
import static accord.utils.Property.qt;
37+
38+
public class SerializePackedTest
39+
{
40+
private static final Gen<int[]> zeros = rs -> new int[rs.nextInt(0, 10)];
41+
private static final Gen<int[]> randomZeroAndPositive = array(rs -> rs.nextInt(-1, Integer.MAX_VALUE) + 1);
42+
private static final Gen<int[]> randomZeroAndPositiveSmall = array(rs -> rs.nextInt(0, 1 << 8));
43+
private static final Gen<int[]> negatives = array(Gens.ints().between(Integer.MIN_VALUE, -1), rs -> rs.nextInt(1, 10));
44+
45+
private static final Gen<int[]> monotonic = rs -> {
46+
int[] array = new int[rs.nextInt(0, 10)];
47+
for (int i = 0; i < array.length; i++)
48+
array[i] = i;
49+
return array;
50+
};
51+
52+
private static final Gen<int[]> zeroAndPositive = rs -> {
53+
if (rs.decide(0.2f)) return monotonic.next(rs);
54+
return randomZeroAndPositive.next(rs);
55+
};
56+
57+
private static final Gen<int[]> zeroAndPositiveSmall = rs -> {
58+
if (rs.decide(0.2f)) return monotonic.next(rs);
59+
return randomZeroAndPositiveSmall.next(rs);
60+
};
61+
62+
private static Gen<int[]> array(Gen.IntGen valueGen)
63+
{
64+
return array(valueGen, rs -> rs.nextInt(0, 10));
65+
}
66+
67+
private static Gen<int[]> array(Gen.IntGen valueGen, Gen.IntGen size)
68+
{
69+
return rs -> {
70+
int[] array = new int[size.nextInt(rs)];
71+
for (int i = 0; i < array.length; i++)
72+
array[i] = valueGen.nextInt(rs);
73+
Arrays.sort(array);
74+
return array;
75+
};
76+
}
77+
78+
@Test
79+
public void serde()
80+
{
81+
@SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) DataOutputBuffer output = new DataOutputBuffer();
82+
qt().forAll(zeroAndPositive).check(array -> Serializers.testSerde(output, PackedSortedSerializer.instance, array));
83+
}
84+
85+
@Test
86+
public void serdeNegative()
87+
{
88+
@SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) DataOutputBuffer output = new DataOutputBuffer();
89+
qt().forAll(negatives).check(array -> {
90+
output.clear();
91+
Assertions.assertThatThrownBy(() -> PackedSortedSerializer.instance.serialize(array, output))
92+
.isInstanceOf(IllegalStateException.class)
93+
.hasMessageContaining("Found a negative value at offset");
94+
});
95+
}
96+
97+
@Test
98+
public void serdeZeros()
99+
{
100+
@SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) DataOutputBuffer output = new DataOutputBuffer();
101+
qt().forAll(zeros).check(array -> Serializers.testSerde(output, PackedSortedSerializer.instance, array));
102+
}
103+
104+
@Test
105+
public void serializerIsSmallerThanSimpleList()
106+
{
107+
qt().forAll(zeroAndPositiveSmall).check(array -> {
108+
var list = SimpleListSerializer.instance.serialize(array);
109+
var packed = PackedSortedSerializer.instance.serialize(array);
110+
111+
Assertions.assertThat(packed.remaining()).isLessThanOrEqualTo(list.remaining());
112+
});
113+
}
114+
115+
public static class PackedSortedSerializer implements UnversionedSerializer<int[]>
116+
{
117+
public static final PackedSortedSerializer instance = new PackedSortedSerializer();
118+
119+
@Override
120+
public void serialize(int[] t, DataOutputPlus out) throws IOException
121+
{
122+
SerializePacked.serializePackedSortedIntsAndLength(t, out);
123+
}
124+
125+
@Override
126+
public int[] deserialize(DataInputPlus in) throws IOException
127+
{
128+
return SerializePacked.deserializePackedSortedIntsAndLength(in);
129+
}
130+
131+
@Override
132+
public long serializedSize(int[] t)
133+
{
134+
return SerializePacked.serializedSizeOfPackedSortedIntsAndLength(t);
135+
}
136+
}
137+
138+
public static class SimpleListSerializer implements UnversionedSerializer<int[]>
139+
{
140+
public static final SimpleListSerializer instance = new SimpleListSerializer();
141+
142+
@Override
143+
public void serialize(int[] t, DataOutputPlus out) throws IOException
144+
{
145+
out.writeUnsignedVInt32(t.length);
146+
for (int i : t)
147+
out.writeVInt32(i);
148+
}
149+
150+
@Override
151+
public int[] deserialize(DataInputPlus in) throws IOException
152+
{
153+
int size = in.readUnsignedVInt32();
154+
int[] array = new int[size];
155+
for (int i = 0; i < size; i++)
156+
array[i] = in.readVInt32();
157+
return array;
158+
}
159+
160+
@Override
161+
public long serializedSize(int[] t)
162+
{
163+
long size = TypeSizes.sizeofUnsignedVInt(t.length);
164+
for (int i = 0; i < t.length; i++)
165+
size += TypeSizes.sizeofVInt(t[i]);
166+
return size;
167+
}
168+
}
169+
}

0 commit comments

Comments
 (0)