Skip to content

Commit 14f9d36

Browse files
pguyotpguyot
committed
AArch64: increase coverage and use cbz/tbz/tbnz
Also simplify {RegOrTuple, '&', Val, '!=', 0} when possible Signed-off-by: Paul Guyot <[email protected]>
1 parent fb34b13 commit 14f9d36

File tree

4 files changed

+895
-140
lines changed

4 files changed

+895
-140
lines changed

‎libs/jit/src/jit_aarch64.erl‎

Lines changed: 103 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -158,12 +158,15 @@
158158
{free, aarch64_register()} | aarch64_register().
159159

160160
-type condition() ::
161-
{aarch64_register(), '<', 0}
162-
| {maybe_free_aarch64_register(), '==', 0}
163-
| {maybe_free_aarch64_register(), '!=', integer()}
161+
{aarch64_register(), '<', integer()}
162+
| {maybe_free_aarch64_register(), '<', aarch64_register()}
163+
| {maybe_free_aarch64_register(), '==', integer()}
164+
| {maybe_free_aarch64_register(), '!=', aarch64_register() | integer()}
165+
| {'(int)', maybe_free_aarch64_register(), '==', integer()}
166+
| {'(int)', maybe_free_aarch64_register(), '!=', aarch64_register() | integer()}
164167
| {'(bool)', maybe_free_aarch64_register(), '==', false}
165168
| {'(bool)', maybe_free_aarch64_register(), '!=', false}
166-
| {maybe_free_aarch64_register(), '&', non_neg_integer(), '!=', 0}.
169+
| {maybe_free_aarch64_register(), '&', non_neg_integer(), '!=', integer()}.
167170

168171
% ctx->e is 0x28
169172
% ctx->x is 0x30
@@ -518,6 +521,21 @@ jump_to_label(
518521
Stream1 = StreamModule:append(Stream0, I1),
519522
State#state{stream = Stream1, branches = [Reloc | AccBranches]}.
520523

524+
%% @private
525+
-spec rewrite_branch_instruction(
526+
jit_aarch64_asm:cc() | {tbz | tbnz, atom(), 0..63} | {cbz, atom()}, integer()
527+
) -> binary().
528+
rewrite_branch_instruction({cbnz, Reg}, Offset) ->
529+
jit_aarch64_asm:cbnz(Reg, Offset);
530+
rewrite_branch_instruction({cbnz_w, Reg}, Offset) ->
531+
jit_aarch64_asm:cbnz_w(Reg, Offset);
532+
rewrite_branch_instruction({tbz, Reg, Bit}, Offset) ->
533+
jit_aarch64_asm:tbz(Reg, Bit, Offset);
534+
rewrite_branch_instruction({tbnz, Reg, Bit}, Offset) ->
535+
jit_aarch64_asm:tbnz(Reg, Bit, Offset);
536+
rewrite_branch_instruction(CC, Offset) when is_atom(CC) ->
537+
jit_aarch64_asm:bcc(CC, Offset).
538+
521539
%%-----------------------------------------------------------------------------
522540
%% @doc Emit an if block, i.e. emit a test of a condition and conditionnally
523541
%% execute a block.
@@ -567,7 +585,7 @@ if_block(
567585
OffsetAfter = StreamModule:offset(Stream2),
568586
%% Patch the conditional branch instruction to jump to the end of the block
569587
BranchOffset = OffsetAfter - (Offset + BranchInstrOffset),
570-
NewBranchInstr = jit_aarch64_asm:bcc(CC, BranchOffset),
588+
NewBranchInstr = rewrite_branch_instruction(CC, BranchOffset),
571589
Stream3 = StreamModule:replace(Stream2, Offset + BranchInstrOffset, NewBranchInstr),
572590
merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs).
573591

@@ -601,7 +619,7 @@ if_else_block(
601619
OffsetAfter = StreamModule:offset(Stream3),
602620
%% Patch the conditional branch to jump to the else block
603621
ElseBranchOffset = OffsetAfter - (Offset + BranchInstrOffset),
604-
NewBranchInstr = jit_aarch64_asm:bcc(CC, ElseBranchOffset),
622+
NewBranchInstr = rewrite_branch_instruction(CC, ElseBranchOffset),
605623
Stream4 = StreamModule:replace(Stream3, Offset + BranchInstrOffset, NewBranchInstr),
606624
%% Build the else block
607625
StateElse = State2#state{
@@ -619,23 +637,41 @@ if_else_block(
619637
Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr),
620638
merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs).
621639

622-
-spec if_block_cond(state(), condition()) -> {state(), jit_aarch64_asm:cc(), non_neg_integer()}.
640+
-spec if_block_cond(state(), condition()) ->
641+
{
642+
state(),
643+
jit_aarch64_asm:cc() | {tbz | tbnz, atom(), 0..63} | {cbz, atom()},
644+
non_neg_integer()
645+
}.
623646
if_block_cond(#state{stream_module = StreamModule, stream = Stream0} = State0, {Reg, '<', 0}) ->
624-
I1 = jit_aarch64_asm:tst(Reg, Reg),
625-
% pl = positive or zero (>=0)
626-
I2 = jit_aarch64_asm:bcc(pl, 0),
647+
I = jit_aarch64_asm:tbz(Reg, 63, 0),
648+
Stream1 = StreamModule:append(Stream0, I),
649+
State1 = State0#state{stream = Stream1},
650+
{State1, {tbz, Reg, 63}, 0};
651+
if_block_cond(
652+
#state{stream_module = StreamModule, stream = Stream0} = State0,
653+
{Reg, '<', Val}
654+
) when is_atom(Reg), is_integer(Val) ->
655+
I1 = jit_aarch64_asm:cmp(Reg, Val),
656+
% ge = greater than or equal
657+
I2 = jit_aarch64_asm:bcc(ge, 0),
627658
Code = <<
628659
I1/binary,
629660
I2/binary
630661
>>,
631662
Stream1 = StreamModule:append(Stream0, Code),
632663
State1 = State0#state{stream = Stream1},
633-
{State1, pl, byte_size(I1)};
664+
{State1, ge, byte_size(I1)};
634665
if_block_cond(
635666
#state{stream_module = StreamModule, stream = Stream0} = State0,
636-
{RegA, '<', RegB}
637-
) when ?IS_GPR(RegA) ->
638-
I1 = jit_aarch64_asm:cmp(RegA, RegB),
667+
{RegOrTuple, '<', RegB}
668+
) when is_atom(RegB) ->
669+
Reg =
670+
case RegOrTuple of
671+
{free, Reg0} -> Reg0;
672+
RegOrTuple -> RegOrTuple
673+
end,
674+
I1 = jit_aarch64_asm:cmp(Reg, RegB),
639675
% ge = greater than or equal
640676
I2 = jit_aarch64_asm:bcc(ge, 0),
641677
Code = <<
@@ -653,17 +689,11 @@ if_block_cond(
653689
{free, Reg0} -> Reg0;
654690
RegOrTuple -> RegOrTuple
655691
end,
656-
I1 = jit_aarch64_asm:tst(Reg, Reg),
657-
% ne = not equal
658-
I2 = jit_aarch64_asm:bcc(ne, 0),
659-
Code = <<
660-
I1/binary,
661-
I2/binary
662-
>>,
663-
Stream1 = StreamModule:append(Stream0, Code),
692+
I = jit_aarch64_asm:cbnz(Reg, 0),
693+
Stream1 = StreamModule:append(Stream0, I),
664694
State1 = if_block_free_reg(RegOrTuple, State0),
665695
State2 = State1#state{stream = Stream1},
666-
{State2, ne, byte_size(I1)};
696+
{State2, {cbnz, Reg}, 0};
667697
if_block_cond(
668698
#state{stream_module = StreamModule, stream = Stream0} = State0, {'(int)', RegOrTuple, '==', 0}
669699
) ->
@@ -672,7 +702,21 @@ if_block_cond(
672702
{free, Reg0} -> Reg0;
673703
RegOrTuple -> RegOrTuple
674704
end,
675-
I1 = jit_aarch64_asm:tst_w(Reg, Reg),
705+
I = jit_aarch64_asm:cbnz_w(Reg, 0),
706+
Stream1 = StreamModule:append(Stream0, I),
707+
State1 = if_block_free_reg(RegOrTuple, State0),
708+
State2 = State1#state{stream = Stream1},
709+
{State2, {cbnz_w, Reg}, 0};
710+
if_block_cond(
711+
#state{stream_module = StreamModule, stream = Stream0} = State0,
712+
{'(int)', RegOrTuple, '==', Val}
713+
) when is_integer(Val) ->
714+
Reg =
715+
case RegOrTuple of
716+
{free, Reg0} -> Reg0;
717+
RegOrTuple -> RegOrTuple
718+
end,
719+
I1 = jit_aarch64_asm:cmp_w(Reg, Val),
676720
I2 = jit_aarch64_asm:bcc(ne, 0),
677721
Code = <<
678722
I1/binary,
@@ -691,11 +735,7 @@ if_block_cond(
691735
{free, Reg0} -> Reg0;
692736
RegOrTuple -> RegOrTuple
693737
end,
694-
I1 =
695-
case Val of
696-
V when is_integer(V) -> jit_aarch64_asm:cmp(Reg, V);
697-
V when is_atom(V) -> jit_aarch64_asm:cmp(Reg, V)
698-
end,
738+
I1 = jit_aarch64_asm:cmp(Reg, Val),
699739
I2 = jit_aarch64_asm:bcc(eq, 0),
700740
Code = <<
701741
I1/binary,
@@ -708,17 +748,13 @@ if_block_cond(
708748
if_block_cond(
709749
#state{stream_module = StreamModule, stream = Stream0} = State0,
710750
{'(int)', RegOrTuple, '!=', Val}
711-
) when is_integer(Val) orelse ?IS_GPR(Val) ->
751+
) when is_integer(Val) ->
712752
Reg =
713753
case RegOrTuple of
714754
{free, Reg0} -> Reg0;
715755
RegOrTuple -> RegOrTuple
716756
end,
717-
I1 =
718-
case Val of
719-
V when is_integer(V) -> jit_aarch64_asm:cmp32(Reg, V);
720-
V when is_atom(V) -> jit_aarch64_asm:cmp32(Reg, V)
721-
end,
757+
I1 = jit_aarch64_asm:cmp_w(Reg, Val),
722758
I2 = jit_aarch64_asm:bcc(eq, 0),
723759
Code = <<
724760
I1/binary,
@@ -731,17 +767,13 @@ if_block_cond(
731767
if_block_cond(
732768
#state{stream_module = StreamModule, stream = Stream0} = State0,
733769
{RegOrTuple, '==', Val}
734-
) when is_integer(Val) orelse ?IS_GPR(Val) ->
770+
) when is_integer(Val) ->
735771
Reg =
736772
case RegOrTuple of
737773
{free, Reg0} -> Reg0;
738774
RegOrTuple -> RegOrTuple
739775
end,
740-
I1 =
741-
case Val of
742-
V when is_integer(V) -> jit_aarch64_asm:cmp(Reg, V);
743-
V when is_atom(V) -> jit_aarch64_asm:cmp(Reg, V)
744-
end,
776+
I1 = jit_aarch64_asm:cmp(Reg, Val),
745777
I2 = jit_aarch64_asm:bcc(ne, 0),
746778
Code = <<
747779
I1/binary,
@@ -753,67 +785,66 @@ if_block_cond(
753785
{State2, ne, byte_size(I1)};
754786
if_block_cond(
755787
#state{stream_module = StreamModule, stream = Stream0} = State0,
756-
{'(int)', RegOrTuple, '==', Val}
757-
) when is_integer(Val) orelse ?IS_GPR(Val) ->
788+
{'(bool)', RegOrTuple, '==', false}
789+
) ->
758790
Reg =
759791
case RegOrTuple of
760792
{free, Reg0} -> Reg0;
761793
RegOrTuple -> RegOrTuple
762794
end,
763-
I1 =
764-
case Val of
765-
V when is_integer(V) -> jit_aarch64_asm:cmp32(Reg, V);
766-
V when is_atom(V) -> jit_aarch64_asm:cmp32(Reg, V)
767-
end,
768-
I2 = jit_aarch64_asm:bcc(ne, 0),
769-
Code = <<
770-
I1/binary,
771-
I2/binary
772-
>>,
773-
Stream1 = StreamModule:append(Stream0, Code),
795+
% Test lowest bit
796+
I = jit_aarch64_asm:tbnz(Reg, 0, 0),
797+
Stream1 = StreamModule:append(Stream0, I),
774798
State1 = if_block_free_reg(RegOrTuple, State0),
775799
State2 = State1#state{stream = Stream1},
776-
{State2, ne, byte_size(I1)};
800+
{State2, {tbnz, Reg, 0}, 0};
777801
if_block_cond(
778802
#state{stream_module = StreamModule, stream = Stream0} = State0,
779-
{'(bool)', RegOrTuple, '==', false}
803+
{'(bool)', RegOrTuple, '!=', false}
780804
) ->
781805
Reg =
782806
case RegOrTuple of
783807
{free, Reg0} -> Reg0;
784808
RegOrTuple -> RegOrTuple
785809
end,
786-
% Test low 8 bits
787-
I1 = jit_aarch64_asm:tst_w(Reg, 16#FF),
788-
I2 = jit_aarch64_asm:bcc(ne, 0),
789-
Code = <<
790-
I1/binary,
791-
I2/binary
792-
>>,
793-
Stream1 = StreamModule:append(Stream0, Code),
810+
% Test lowest bit
811+
I = jit_aarch64_asm:tbz(Reg, 0, 0),
812+
Stream1 = StreamModule:append(Stream0, I),
794813
State1 = if_block_free_reg(RegOrTuple, State0),
795814
State2 = State1#state{stream = Stream1},
796-
{State2, ne, byte_size(I1)};
815+
{State2, {tbz, Reg, 0}, 0};
797816
if_block_cond(
798-
#state{stream_module = StreamModule, stream = Stream0} = State0,
799-
{'(bool)', RegOrTuple, '!=', false}
817+
#state{
818+
stream_module = StreamModule,
819+
stream = Stream0,
820+
available_regs = [Temp | _]
821+
} = State0,
822+
{RegOrTuple, '&', Val, '!=', 0}
800823
) ->
801824
Reg =
802825
case RegOrTuple of
803826
{free, Reg0} -> Reg0;
804827
RegOrTuple -> RegOrTuple
805828
end,
806-
% Test low 8 bits
807-
I1 = jit_aarch64_asm:tst_w(Reg, 16#FF),
829+
% Test bits
830+
TestCode =
831+
try
832+
jit_aarch64_asm:tst(Reg, Val)
833+
catch
834+
error:{unencodable_immediate, Val} ->
835+
TestCode0 = jit_aarch64_asm:mov(Temp, Val),
836+
TestCode1 = jit_aarch64_asm:tst(Reg, Temp),
837+
<<TestCode0/binary, TestCode1/binary>>
838+
end,
808839
I2 = jit_aarch64_asm:bcc(eq, 0),
809840
Code = <<
810-
I1/binary,
841+
TestCode/binary,
811842
I2/binary
812843
>>,
813844
Stream1 = StreamModule:append(Stream0, Code),
814845
State1 = if_block_free_reg(RegOrTuple, State0),
815846
State2 = State1#state{stream = Stream1},
816-
{State2, eq, byte_size(I1)};
847+
{State2, eq, byte_size(TestCode)};
817848
if_block_cond(
818849
#state{
819850
stream_module = StreamModule,
@@ -853,53 +884,7 @@ if_block_cond(
853884
Stream3 = StreamModule:append(Stream2, I3),
854885
State3 = State1#state{stream = Stream3},
855886
State4 = if_block_free_reg(RegTuple, State3),
856-
{State4, eq, OffsetAfter - OffsetBefore};
857-
if_block_cond(
858-
#state{
859-
stream_module = StreamModule,
860-
stream = Stream0
861-
} = State0,
862-
{RegOrTuple, '&', Val}
863-
) ->
864-
Reg =
865-
case RegOrTuple of
866-
{free, Reg0} -> Reg0;
867-
RegOrTuple -> RegOrTuple
868-
end,
869-
% Test bits
870-
I1 = jit_aarch64_asm:tst(Reg, Val),
871-
I2 = jit_aarch64_asm:bcc(eq, 0),
872-
Code = <<
873-
I1/binary,
874-
I2/binary
875-
>>,
876-
Stream1 = StreamModule:append(Stream0, Code),
877-
State1 = if_block_free_reg(RegOrTuple, State0),
878-
State2 = State1#state{stream = Stream1},
879-
{State2, eq, byte_size(I1)};
880-
if_block_cond(
881-
#state{
882-
stream_module = StreamModule,
883-
stream = Stream0
884-
} = State0,
885-
{'(bool)', RegOrTuple, '&', Val}
886-
) ->
887-
Reg =
888-
case RegOrTuple of
889-
{free, Reg0} -> Reg0;
890-
RegOrTuple -> RegOrTuple
891-
end,
892-
% Test 8-bit value
893-
I1 = jit_aarch64_asm:tst_w(Reg, Val),
894-
I2 = jit_aarch64_asm:bcc(eq, 0),
895-
Code = <<
896-
I1/binary,
897-
I2/binary
898-
>>,
899-
Stream1 = StreamModule:append(Stream0, Code),
900-
State1 = if_block_free_reg(RegOrTuple, State0),
901-
State2 = State1#state{stream = Stream1},
902-
{State2, eq, byte_size(I1)}.
887+
{State4, eq, OffsetAfter - OffsetBefore}.
903888

904889
-spec if_block_free_reg(aarch64_register() | {free, aarch64_register()}, state()) -> state().
905890
if_block_free_reg({free, Reg}, State0) ->

0 commit comments

Comments
 (0)