Skip to content

Commit f4c1755

Browse files
jnmtjnmt
committed
Fix to handle FLOAT and BLOB data types in PutToMutable function (#297)
1 parent 8834b26 commit f4c1755

File tree

4 files changed

+184
-45
lines changed

4 files changed

+184
-45
lines changed

‎generic-contracts/scripts/objects-table-schema.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,14 @@
1111
"object_id": "TEXT",
1212
"version": "TEXT",
1313
"status": "INT",
14-
"registered_at": "BIGINT"
14+
"registered_at": "BIGINT",
15+
"column_boolean": "BOOLEAN",
16+
"column_bigint": "BIGINT",
17+
"column_float": "FLOAT",
18+
"column_double": "DOUBLE",
19+
"column_text": "TEXT",
20+
"column_blob": "BLOB"
1521
},
1622
"compaction-strategy": "LCS"
1723
}
1824
}
19-

‎generic-contracts/src/integration-test/java/com/scalar/dl/genericcontracts/GenericContractEndToEndTest.java‎

Lines changed: 157 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,16 @@
6666
import com.scalar.db.common.error.CoreError;
6767
import com.scalar.db.config.DatabaseConfig;
6868
import com.scalar.db.exception.storage.ExecutionException;
69+
import com.scalar.db.io.BigIntColumn;
70+
import com.scalar.db.io.BlobColumn;
71+
import com.scalar.db.io.BooleanColumn;
72+
import com.scalar.db.io.Column;
73+
import com.scalar.db.io.DataType;
74+
import com.scalar.db.io.DoubleColumn;
75+
import com.scalar.db.io.FloatColumn;
76+
import com.scalar.db.io.IntColumn;
6977
import com.scalar.db.io.Key;
78+
import com.scalar.db.io.TextColumn;
7079
import com.scalar.db.schemaloader.SchemaLoader;
7180
import com.scalar.db.schemaloader.SchemaLoaderException;
7281
import com.scalar.db.service.StorageFactory;
@@ -115,9 +124,6 @@ public class GenericContractEndToEndTest {
115124
private static final String ASSET_ID = "id";
116125
private static final String ASSET_AGE = "age";
117126
private static final String ASSET_OUTPUT = "output";
118-
private static final String DATA_TYPE_INT = "INT";
119-
private static final String DATA_TYPE_BIGINT = "BIGINT";
120-
private static final String DATA_TYPE_TEXT = "TEXT";
121127

122128
private static final String JDBC_TRANSACTION_MANAGER = "jdbc";
123129
private static final String PROP_STORAGE = "scalardb.storage";
@@ -241,6 +247,17 @@ public class GenericContractEndToEndTest {
241247
private static final String SOME_COLUMN_NAME_2 = "version";
242248
private static final String SOME_COLUMN_NAME_3 = "status";
243249
private static final String SOME_COLUMN_NAME_4 = "registered_at";
250+
private static final String SOME_COLUMN_NAME_BOOLEAN = "column_boolean";
251+
private static final String SOME_COLUMN_NAME_BIGINT = "column_bigint";
252+
private static final String SOME_COLUMN_NAME_FLOAT = "column_float";
253+
private static final String SOME_COLUMN_NAME_DOUBLE = "column_double";
254+
private static final String SOME_COLUMN_NAME_TEXT = "column_text";
255+
private static final String SOME_COLUMN_NAME_BLOB = "column_blob";
256+
private static final boolean SOME_BOOLEAN_VALUE = false;
257+
private static final long SOME_BIGINT_VALUE = BigIntColumn.MAX_VALUE;
258+
private static final float SOME_FLOAT_VALUE = Float.MAX_VALUE;
259+
private static final double SOME_DOUBLE_VALUE = Double.MAX_VALUE;
260+
private static final byte[] SOME_BLOB_VALUE = {1, 2, 3, 4, 5};
244261
private static final String SOME_COLLECTION_ID = "set";
245262
private static final ArrayNode SOME_DEFAULT_OBJECT_IDS =
246263
mapper.createArrayNode().add("object1").add("object2").add("object3").add("object4");
@@ -432,41 +449,88 @@ private void prepareCollection() {
432449
prepareCollection(clientService);
433450
}
434451

435-
private JsonNode createColumn(String name, int value) {
452+
private JsonNode createColumn(Column<?> column) {
453+
ObjectNode jsonColumn =
454+
mapper
455+
.createObjectNode()
456+
.put(COLUMN_NAME, column.getName())
457+
.put(DATA_TYPE, column.getDataType().name());
458+
459+
switch (column.getDataType()) {
460+
case BOOLEAN:
461+
jsonColumn.put(VALUE, column.getBooleanValue());
462+
break;
463+
case INT:
464+
jsonColumn.put(VALUE, column.getIntValue());
465+
break;
466+
case BIGINT:
467+
jsonColumn.put(VALUE, column.getBigIntValue());
468+
break;
469+
case FLOAT:
470+
jsonColumn.put(VALUE, column.getFloatValue());
471+
break;
472+
case DOUBLE:
473+
jsonColumn.put(VALUE, column.getDoubleValue());
474+
break;
475+
case TEXT:
476+
jsonColumn.put(VALUE, column.getTextValue());
477+
break;
478+
case BLOB:
479+
jsonColumn.put(VALUE, column.getBlobValueAsBytes());
480+
break;
481+
default:
482+
throw new IllegalArgumentException("Invalid data type: " + column.getDataType());
483+
}
484+
485+
return jsonColumn;
486+
}
487+
488+
private JsonNode createColumn(String columnName, DataType dataType, String value) {
436489
return mapper
437490
.createObjectNode()
438-
.put(COLUMN_NAME, name)
491+
.put(COLUMN_NAME, columnName)
439492
.put(VALUE, value)
440-
.put(DATA_TYPE, DATA_TYPE_INT);
493+
.put(DATA_TYPE, dataType.name());
441494
}
442495

443-
private JsonNode createColumn(String name, long value) {
496+
private JsonNode createNullColumn(String columnName, DataType dataType) {
444497
return mapper
445498
.createObjectNode()
446-
.put(COLUMN_NAME, name)
447-
.put(VALUE, value)
448-
.put(DATA_TYPE, DATA_TYPE_BIGINT);
499+
.put(COLUMN_NAME, columnName)
500+
.put(DATA_TYPE, dataType.name())
501+
.set(VALUE, null);
449502
}
450503

451-
private JsonNode createColumn(String name, String value) {
504+
private ArrayNode createColumns(int status) {
452505
return mapper
453-
.createObjectNode()
454-
.put(COLUMN_NAME, name)
455-
.put(VALUE, value)
456-
.put(DATA_TYPE, DATA_TYPE_TEXT);
506+
.createArrayNode()
507+
.add(createColumn(IntColumn.of(SOME_COLUMN_NAME_3, status)))
508+
.add(createColumn(BigIntColumn.of(SOME_COLUMN_NAME_4, SOME_BIGINT_VALUE)))
509+
.add(createColumn(BooleanColumn.of(SOME_COLUMN_NAME_BOOLEAN, SOME_BOOLEAN_VALUE)))
510+
.add(createColumn(BigIntColumn.of(SOME_COLUMN_NAME_BIGINT, SOME_BIGINT_VALUE)))
511+
.add(createColumn(FloatColumn.of(SOME_COLUMN_NAME_FLOAT, SOME_FLOAT_VALUE)))
512+
.add(createColumn(DoubleColumn.of(SOME_COLUMN_NAME_DOUBLE, SOME_DOUBLE_VALUE)))
513+
.add(createColumn(BlobColumn.of(SOME_COLUMN_NAME_BLOB, SOME_BLOB_VALUE)));
514+
}
515+
516+
private ArrayNode createNullColumns() {
517+
return mapper
518+
.createArrayNode()
519+
.add(createNullColumn(SOME_COLUMN_NAME_3, DataType.INT))
520+
.add(createNullColumn(SOME_COLUMN_NAME_4, DataType.BIGINT))
521+
.add(createNullColumn(SOME_COLUMN_NAME_BOOLEAN, DataType.BOOLEAN))
522+
.add(createNullColumn(SOME_COLUMN_NAME_BIGINT, DataType.BIGINT))
523+
.add(createNullColumn(SOME_COLUMN_NAME_FLOAT, DataType.FLOAT))
524+
.add(createNullColumn(SOME_COLUMN_NAME_DOUBLE, DataType.DOUBLE))
525+
.add(createNullColumn(SOME_COLUMN_NAME_TEXT, DataType.TEXT))
526+
.add(createNullColumn(SOME_COLUMN_NAME_BLOB, DataType.BLOB));
457527
}
458528

459-
private JsonNode createFunctionArguments(
460-
String objectId, String version, int status, long registeredAt) {
529+
private ObjectNode createFunctionArguments(String objectId, String version, ArrayNode columns) {
461530
ArrayNode partitionKey =
462-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_1, objectId));
531+
mapper.createArrayNode().add(createColumn(TextColumn.of(SOME_COLUMN_NAME_1, objectId)));
463532
ArrayNode clusteringKey =
464-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_2, version));
465-
ArrayNode columns =
466-
mapper
467-
.createArrayNode()
468-
.add(createColumn(SOME_COLUMN_NAME_3, status))
469-
.add(createColumn(SOME_COLUMN_NAME_4, registeredAt));
533+
mapper.createArrayNode().add(createColumn(TextColumn.of(SOME_COLUMN_NAME_2, version)));
470534

471535
ObjectNode arguments = mapper.createObjectNode();
472536
arguments.put(NAMESPACE, SOME_FUNCTION_NAMESPACE);
@@ -478,6 +542,14 @@ private JsonNode createFunctionArguments(
478542
return arguments;
479543
}
480544

545+
private ObjectNode createFunctionArguments(String objectId, String version, int status) {
546+
return createFunctionArguments(objectId, version, createColumns(status));
547+
}
548+
549+
private ObjectNode createFunctionArgumentsWithNullColumns(String objectId, String version) {
550+
return createFunctionArguments(objectId, version, createNullColumns());
551+
}
552+
481553
private void addObjectsToCollection(
482554
GenericContractClientService clientService, String collectionId, ArrayNode objectIds) {
483555
JsonNode arguments =
@@ -749,9 +821,8 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
749821
.put(OBJECT_ID, SOME_OBJECT_ID)
750822
.put(HASH_VALUE, SOME_HASH_VALUE_1)
751823
.set(METADATA, SOME_METADATA_1);
752-
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0, 1L);
753-
JsonNode functionArguments1 =
754-
createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_1, 1, 1234567890123L);
824+
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0);
825+
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_1, 1);
755826
Scan scan =
756827
Scan.newBuilder()
757828
.namespace(SOME_FUNCTION_NAMESPACE)
@@ -773,11 +844,63 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
773844
assertThat(results.get(0).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
774845
assertThat(results.get(0).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_0);
775846
assertThat(results.get(0).getInt(SOME_COLUMN_NAME_3)).isEqualTo(0);
776-
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(1L);
847+
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(SOME_BIGINT_VALUE);
848+
assertThat(results.get(0).getBoolean(SOME_COLUMN_NAME_BOOLEAN)).isEqualTo(SOME_BOOLEAN_VALUE);
849+
assertThat(results.get(0).getBigInt(SOME_COLUMN_NAME_BIGINT)).isEqualTo(SOME_BIGINT_VALUE);
850+
assertThat(results.get(0).getFloat(SOME_COLUMN_NAME_FLOAT)).isEqualTo(SOME_FLOAT_VALUE);
851+
assertThat(results.get(0).getDouble(SOME_COLUMN_NAME_DOUBLE)).isEqualTo(SOME_DOUBLE_VALUE);
852+
assertThat(results.get(0).getBlobAsBytes(SOME_COLUMN_NAME_BLOB)).isEqualTo(SOME_BLOB_VALUE);
777853
assertThat(results.get(1).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
778854
assertThat(results.get(1).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_1);
779855
assertThat(results.get(1).getInt(SOME_COLUMN_NAME_3)).isEqualTo(1);
780-
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(1234567890123L);
856+
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_4)).isEqualTo(SOME_BIGINT_VALUE);
857+
assertThat(results.get(1).getBoolean(SOME_COLUMN_NAME_BOOLEAN)).isEqualTo(SOME_BOOLEAN_VALUE);
858+
assertThat(results.get(1).getBigInt(SOME_COLUMN_NAME_BIGINT)).isEqualTo(SOME_BIGINT_VALUE);
859+
assertThat(results.get(1).getFloat(SOME_COLUMN_NAME_FLOAT)).isEqualTo(SOME_FLOAT_VALUE);
860+
assertThat(results.get(1).getDouble(SOME_COLUMN_NAME_DOUBLE)).isEqualTo(SOME_DOUBLE_VALUE);
861+
assertThat(results.get(1).getBlobAsBytes(SOME_COLUMN_NAME_BLOB)).isEqualTo(SOME_BLOB_VALUE);
862+
} catch (IOException e) {
863+
throw new RuntimeException(e);
864+
}
865+
}
866+
867+
@Test
868+
public void putObject_FunctionArgumentsWithNullColumnsGiven_ShouldPutRecordToFunctionTable()
869+
throws ExecutionException {
870+
// Arrange
871+
JsonNode contractArguments =
872+
mapper
873+
.createObjectNode()
874+
.put(OBJECT_ID, SOME_OBJECT_ID)
875+
.put(HASH_VALUE, SOME_HASH_VALUE_0)
876+
.set(METADATA, SOME_METADATA_0);
877+
JsonNode functionArguments =
878+
createFunctionArgumentsWithNullColumns(SOME_OBJECT_ID, SOME_VERSION_ID_0);
879+
Scan scan =
880+
Scan.newBuilder()
881+
.namespace(SOME_FUNCTION_NAMESPACE)
882+
.table(SOME_FUNCTION_TABLE)
883+
.partitionKey(Key.ofText(SOME_COLUMN_NAME_1, SOME_OBJECT_ID))
884+
.build();
885+
886+
// Act
887+
clientService.executeContract(
888+
ID_OBJECT_PUT, contractArguments, ID_OBJECT_PUT_MUTABLE, functionArguments);
889+
890+
// Assert
891+
try (Scanner scanner = storage.scan(scan)) {
892+
List<Result> results = scanner.all();
893+
assertThat(results).hasSize(1);
894+
assertThat(results.get(0).getText(SOME_COLUMN_NAME_1)).isEqualTo(SOME_OBJECT_ID);
895+
assertThat(results.get(0).getText(SOME_COLUMN_NAME_2)).isEqualTo(SOME_VERSION_ID_0);
896+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_3)).isTrue();
897+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_4)).isTrue();
898+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BOOLEAN)).isTrue();
899+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BIGINT)).isTrue();
900+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_FLOAT)).isTrue();
901+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_DOUBLE)).isTrue();
902+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_TEXT)).isTrue();
903+
assertThat(results.get(0).isNull(SOME_COLUMN_NAME_BLOB)).isTrue();
781904
} catch (IOException e) {
782905
throw new RuntimeException(e);
783906
}
@@ -800,7 +923,9 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
800923
.put(TABLE, "foo")
801924
.set(
802925
PARTITION_KEY,
803-
mapper.createArrayNode().add(createColumn(SOME_COLUMN_NAME_1, SOME_OBJECT_ID)));
926+
mapper
927+
.createArrayNode()
928+
.add(createColumn(TextColumn.of(SOME_COLUMN_NAME_1, SOME_OBJECT_ID))));
804929

805930
// Act Assert
806931
assertThatThrownBy(
@@ -832,8 +957,8 @@ public void putObject_FunctionArgumentsGiven_ShouldPutRecordToFunctionTable()
832957
.put(OBJECT_ID, SOME_OBJECT_ID)
833958
.put(HASH_VALUE, SOME_HASH_VALUE_1)
834959
.set(METADATA, SOME_METADATA_1);
835-
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0, 1L);
836-
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 1, 1L);
960+
JsonNode functionArguments0 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 0);
961+
JsonNode functionArguments1 = createFunctionArguments(SOME_OBJECT_ID, SOME_VERSION_ID_0, 1);
837962
Put put =
838963
Put.newBuilder()
839964
.namespace(SOME_FUNCTION_NAMESPACE)

‎generic-contracts/src/main/java/com/scalar/dl/genericcontracts/object/PutToMutableDatabase.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,10 @@ private Column<?> getColumn(JsonNode jsonColumn) {
162162
}
163163

164164
if (dataType.equals(DataType.FLOAT)) {
165-
if (!value.isFloat()) {
165+
// The JSON deserializer does not distinguish between float and double values; all JSON
166+
// numbers with a decimal point are deserialized as double. Therefore, we check for isDouble()
167+
// here even for FLOAT columns.
168+
if (!value.isDouble()) {
166169
throw new ContractContextException(Constants.INVALID_PUT_MUTABLE_FUNCTION_ARGUMENT_FORMAT);
167170
}
168171
return FloatColumn.of(columnName, value.floatValue());
@@ -183,7 +186,9 @@ private Column<?> getColumn(JsonNode jsonColumn) {
183186
}
184187

185188
if (dataType.equals(DataType.BLOB)) {
186-
if (!value.isBinary()) {
189+
// BLOB data is expected as a Base64-encoded string due to JSON limitations. JSON cannot
190+
// represent binary data directly, so BLOBs must be provided as Base64-encoded strings.
191+
if (!value.isTextual()) {
187192
throw new ContractContextException(Constants.INVALID_PUT_MUTABLE_FUNCTION_ARGUMENT_FORMAT);
188193
}
189194
try {

‎generic-contracts/src/test/java/com/scalar/dl/genericcontracts/object/PutToMutableDatabaseTest.java‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public class PutToMutableDatabaseTest {
4646
private static final String SOME_DOUBLE_COLUMN_NAME = "double_column";
4747
private static final double SOME_DOUBLE_COLUMN_VALUE = 1.23;
4848
private static final String SOME_BLOB_COLUMN_NAME = "blob_column";
49-
private static final byte[] SOME_BLOB_COLUMN_VALUE = {10, 20, 30};
49+
private static final byte[] SOME_BLOB_COLUMN_VALUE = {1, 2, 3, 4, 5};
50+
private static final String SOME_BLOB_COLUMN_TEXT = "AQIDBAU="; // Base64 of {1, 2, 3, 4, 5}
5051
private static final JsonNode SOME_TEXT_COLUMN1 =
5152
mapper
5253
.createObjectNode()
@@ -81,7 +82,10 @@ public class PutToMutableDatabaseTest {
8182
mapper
8283
.createObjectNode()
8384
.put(Constants.COLUMN_NAME, SOME_FLOAT_COLUMN_NAME)
84-
.put(Constants.VALUE, SOME_FLOAT_COLUMN_VALUE)
85+
.put(
86+
Constants.VALUE,
87+
Double.valueOf(
88+
SOME_FLOAT_COLUMN_VALUE)) // float is always converted to double in function args
8589
.put(Constants.DATA_TYPE, "FLOAT");
8690
private static final JsonNode SOME_DOUBLE_COLUMN =
8791
mapper
@@ -93,7 +97,7 @@ public class PutToMutableDatabaseTest {
9397
mapper
9498
.createObjectNode()
9599
.put(Constants.COLUMN_NAME, SOME_BLOB_COLUMN_NAME)
96-
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
100+
.put(Constants.VALUE, SOME_BLOB_COLUMN_TEXT)
97101
.put(Constants.DATA_TYPE, "BLOB");
98102
private static final JsonNode SOME_NULL_COLUMN =
99103
mapper
@@ -481,9 +485,9 @@ public void invoke_ColumnsWithUnmatchedTypeGiven_ShouldThrowContractContextExcep
481485
mapper
482486
.createObjectNode()
483487
.put(Constants.COLUMN_NAME, SOME_FLOAT_COLUMN_NAME)
484-
.put(Constants.VALUE, SOME_DOUBLE_COLUMN_VALUE)
488+
.put(Constants.VALUE, SOME_INT_COLUMN_VALUE)
485489
.put(Constants.DATA_TYPE, "FLOAT"),
486-
"DOUBLE value with FLOAT data type")
490+
"INT value with FLOAT data type")
487491
.put(
488492
mapper
489493
.createObjectNode()
@@ -495,16 +499,16 @@ public void invoke_ColumnsWithUnmatchedTypeGiven_ShouldThrowContractContextExcep
495499
mapper
496500
.createObjectNode()
497501
.put(Constants.COLUMN_NAME, SOME_TEXT_COLUMN1_NAME)
498-
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
502+
.put(Constants.VALUE, SOME_INT_COLUMN_VALUE)
499503
.put(Constants.DATA_TYPE, "TEXT"),
500-
"BLOB value with TEXT data type")
504+
"INT value with TEXT data type")
501505
.put(
502506
mapper
503507
.createObjectNode()
504508
.put(Constants.COLUMN_NAME, SOME_BLOB_COLUMN_NAME)
505-
.put(Constants.VALUE, SOME_TEXT_COLUMN1_VALUE)
509+
.put(Constants.VALUE, SOME_BLOB_COLUMN_VALUE)
506510
.put(Constants.DATA_TYPE, "BLOB"),
507-
"TEXT value with BLOB data type");
511+
"BLOB value with BLOB data type");
508512

509513
// Act Assert
510514
builder

0 commit comments

Comments
 (0)