Merge pull request #78 from helium/adt/less-serializing

Don't term_to_binary as much when serializing
helium · Jul 11, 2022 · 16a75ba · 16a75ba
2 parents f4b8137 + 787dd16
commit 16a75ba
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 64 deletions.
diff --git a/src/hbbft.erl b/src/hbbft.erl
@@ -694,13 +694,13 @@ encrypt(SK, Bin) ->
     tc_ciphertext:serialize(tc_key_share:encrypt(SK, Bin)).
 
 -spec serialize(hbbft_data()) ->
-    {#{atom() => binary() | map()}, binary() | tc_key_share:tc_key_share()}.
+    {#{atom() => term()}, binary() | tc_key_share:tc_key_share()}.
 serialize(Data) ->
     %% serialize the SK unless explicitly told not to
     serialize(Data, true).
 
 -spec serialize(hbbft_data(), boolean()) ->
-    {#{atom() => binary() | map()}, binary() | tc_key_share:tc_key_share()}.
+    {#{atom() => term()}, binary() | tc_key_share:tc_key_share()}.
 serialize(#hbbft_data{key_share = SK} = Data, false) ->
     %% dont serialize the private key
     {serialize_hbbft_data(Data), SK};
@@ -709,14 +709,7 @@ serialize(#hbbft_data{key_share = SK} = Data, true) ->
     {serialize_hbbft_data(Data), tc_key_share:serialize(SK)}.
 
 -spec deserialize(#{atom() => binary() | map()}, tc_key_share:tc_key_share()) -> hbbft_data().
-deserialize(M0, SK) ->
-    M = maps:map(
-        fun
-            (acs, V) -> V;
-            (_K, V) -> binary_to_term(V)
-        end,
-        M0
-    ),
+deserialize(M, SK) ->
     #{
         batch_size := BatchSize,
         n := N,
@@ -774,7 +767,7 @@ deserialize(M0, SK) ->
         stamps = Stamps
     }.
 
--spec serialize_hbbft_data(hbbft_data()) -> #{atom() => binary() | map()}.
+-spec serialize_hbbft_data(hbbft_data()) -> #{atom() => term()}.
 serialize_hbbft_data(#hbbft_data{
     batch_size = BatchSize,
     n = N,
@@ -797,38 +790,31 @@ serialize_hbbft_data(#hbbft_data{
     stampfun = Stampfun,
     stamps = Stamps
 }) ->
-    M = #{
-        batch_size => BatchSize,
-        n => N,
-        f => F,
-        round => Round,
-        max_buf => MaxBuf,
-        acs => hbbft_acs:serialize(ACSData),
-        acs_init => ACSInit,
-        sent_txns => SentTxns,
-        decrypted => Decrypted,
-        j => J,
-        sent_sig => SentSig,
-        acs_results => ACSResults,
-        enc_keys => maps:map(fun(_, Ciphertext) -> tc_ciphertext:serialize(Ciphertext) end, EncKeys),
-        dec_shares => maps:map(
-            fun(_, {Valid, Share}) -> {Valid, hbbft_utils:dec_share_to_binary(Share)} end,
-            DecShares
-        ),
-        sig_shares => maps:map(fun(_, V) -> hbbft_utils:sig_share_to_binary(V) end, SigShares),
-        thingtosign => ThingToSign,
-        failed_combine => FailedCombine,
-        failed_decrypt => FailedDecrypt,
-        stampfun => Stampfun,
-        stamps => Stamps
-    },
-    maps:map(
-        fun
-            (acs, V) -> V;
-            (_K, V) -> term_to_binary(V, [compressed])
-        end,
-        M
-    ).
+    #{
+      batch_size => BatchSize,
+      n => N,
+      f => F,
+      round => Round,
+      max_buf => MaxBuf,
+      acs => hbbft_acs:serialize(ACSData),
+      acs_init => ACSInit,
+      sent_txns => SentTxns,
+      decrypted => Decrypted,
+      j => J,
+      sent_sig => SentSig,
+      acs_results => ACSResults,
+      enc_keys => maps:map(fun(_, Ciphertext) -> tc_ciphertext:serialize(Ciphertext) end, EncKeys),
+      dec_shares => maps:map(
+                      fun(_, {Valid, Share}) -> {Valid, hbbft_utils:dec_share_to_binary(Share)} end,
+                      DecShares
+                     ),
+      sig_shares => maps:map(fun(_, V) -> hbbft_utils:sig_share_to_binary(V) end, SigShares),
+      thingtosign => ThingToSign,
+      failed_combine => FailedCombine,
+      failed_decrypt => FailedDecrypt,
+      stampfun => Stampfun,
+      stamps => Stamps
+     }.
 
 -spec is_serialized(hbbft_data() | #{atom() => binary() | map()}) -> boolean().
 is_serialized(Data) when is_map(Data) -> true;
@@ -933,6 +919,9 @@ buf_insert_test_() ->
     [
         ?_assertMatch({[0], 1}, add_to_buffer([], 0, fun(_) -> true end)),
         ?_assertMatch({[0], 1}, add_to_buffer([], 0, fun(_) -> false end)),
+        ?_assertMatch({[0, 1], 2}, add_to_buffer([0], 1, fun(_) -> true end)),
+        ?_assertMatch({[1, 0], 1}, add_to_buffer([0], 1, fun(_) -> false end)),
+        ?_assertMatch({[a, b, c, c, a, b, a], 4}, add_to_buffer([a, b, c, a, b, a], c, fun(E) -> E /= a andalso E /= b end)),
 
         ?_assertMatch({[0, 1], 1}, add_to_buffer([1], 0, fun(X) -> X < 1 end)),
         ?_assertMatch({[0, 1], 1}, add_to_buffer([1], 0, fun(X) -> X > 1 end)),

diff --git a/src/hbbft_acs.erl b/src/hbbft_acs.erl
@@ -246,20 +246,19 @@ store_rbc_result(Data, I, Result) ->
 sort_bba_msgs(Msgs) ->
     lists:sort(fun({{bba, _}, BBA1}, {{bba, _}, BBA2}) ->
                        hbbft_bba:sort_msgs(BBA1, BBA2);
-                  ({{bba, _}, _}, _) ->
+                 ({{bba, _}, _}, _) ->
                        false;
                   (_, {{bba, _}, _}) ->
                        true;
                   (_, _) ->
                        false
                end, Msgs).
 
--spec serialize(acs_data()) -> #{atom() => #{} | binary()}.
+-spec serialize(acs_data()) -> #{atom() => term()}.
 serialize(#acs_data{done=Done, n=N, f=F, j=J, rbc=RBCMap, bba=BBAMap}) ->
-    M = #{done => Done, n => N, f => F, j => J},
-    M1 = maps:map(fun(_K, V) -> term_to_binary(V, [compressed]) end, M),
-    M1#{rbc => serialize_state(RBCMap, rbc),
-        bba => serialize_state(BBAMap, bba)}.
+    #{done => Done, n => N, f => F, j => J,
+      rbc => serialize_state(RBCMap, rbc),
+      bba => serialize_state(BBAMap, bba)}.
 
 -spec deserialize(acs_serialized_data() | #{}, tc_key_share:tc_key_share()) -> acs_data().
 deserialize(#acs_serialized_data{done=Done, n=N, f=F, j=J, rbc=RBCMap, bba=BBAMap}, SK) ->
@@ -269,8 +268,7 @@ deserialize(#acs_serialized_data{done=Done, n=N, f=F, j=J, rbc=RBCMap, bba=BBAMa
 deserialize(M, SK) ->
     {RBCMap, M1} = maps:take(rbc, M),
     {BBAMap, M2} = maps:take(bba, M1),
-    M3 = maps:map(fun(_K, V) -> binary_to_term(V) end, M2),
-    #{done :=  Done, n :=  N, f :=  F, j :=  J} = M3,
+    #{done :=  Done, n :=  N, f :=  F, j :=  J} = M2,
     #acs_data{done = Done, n = N, f = F, j = J,
               rbc = deserialize_state(RBCMap, rbc),
               bba = deserialize_state(BBAMap, bba, SK)}.
@@ -325,10 +323,9 @@ deserialize_rbc_state(BinRec) when is_binary(BinRec) ->
         binary_to_term(BinRec),
     #rbc_state{rbc_data=RBCData, result=Result}.
 
--spec serialize_bba_state(bba_state()) -> binary().
+-spec serialize_bba_state(bba_state()) -> #bba_serialized_state{}.
 serialize_bba_state(#bba_state{bba_data=BBAData, input=Input, result=Result}) ->
-    term_to_binary(#bba_serialized_state{bba_data=hbbft_bba:serialize(BBAData), input=Input, result=Result},
-                   [compressed]).
+    #bba_serialized_state{bba_data=hbbft_bba:serialize(BBAData), input=Input, result=Result}.
 
 -spec deserialize_bba_state(bba_serialized_state() | #{}, tc_key_share:tc_key_share()) -> bba_state().
 deserialize_bba_state(#bba_serialized_state{bba_data=BBAData, input=Input, result=Result}, SK) ->

diff --git a/src/hbbft_rbc.erl b/src/hbbft_rbc.erl
@@ -280,18 +280,18 @@ hash_key(Map) ->
               end, #{}, Map).
 
 serialize(#rbc_data{stripes=Stripes}=Data) when Stripes /= #{} ->
-    #{rbc_data => term_to_binary(Data#rbc_data{stripes=#{}}), stripes =>
-      maps:map(fun(_K, V) ->
-                       maps:map(fun(_K2, {Index, Size, Shard}) ->
-                                        %% encode as u32 because rocks seems to like dropping smakk keys??
-                                        #{index => <<Index:32/integer>>, size => <<Size:32/integer>>, shard => Shard}
-                                end, V)
-               end, Stripes)};
+    #{rbc_data => Data#rbc_data{stripes=#{}},
+      stripes =>
+        maps:map(fun(_K, V) ->
+                         maps:map(fun(_K2, {Index, Size, Shard}) ->
+                                          %% encode as u32 because rocks seems to like dropping smakk keys??
+                                          #{index => <<Index:32/integer>>, size => <<Size:32/integer>>, shard => Shard}
+                                  end, V)
+                 end, Stripes)};
 serialize(#rbc_data{}=Data) ->
-    #{rbc_data => term_to_binary(Data)}.
+    #{rbc_data => Data}.
 
-deserialize(#{rbc_data := BinData}=Map) ->
-    Data = binary_to_term(BinData),
+deserialize(#{rbc_data := Data}=Map) ->
     Data#rbc_data{stripes=maps:map(fun(_K, V) ->
                        maps:fold(fun(K2, #{index := <<Index:8/integer>>, size := <<Size:32/integer>>, shard := Shard}, Acc) ->
                                         %% legacy decoding