MDEV-37541 Race of rolling back and committing transaction to binlog

This is the 1st of two commits to demonstrate the failure which is
that a rolling back transaction while serializes its completion in
Engine before a contender transaction the two are logged into binary
log in reverse order.

Author: emoonrain

mysql-test/suite/binlog/r/binlog_mdev37541.result

@@ -0,0 +1,37 @@
+create table t1 (a int primary key, b text) engine=innodb;
+connect trx1_rollback,localhost,root,,;
+CREATE TABLE t_x (a int) engine=MEMORY;
+SET binlog_format=row;
+CREATE TEMPORARY TABLE tt_tmp ( id INT ) ENGINE = Memory;
+BEGIN;
+insert into t_x values (1);
+drop temporary table tt_tmp;
+insert into t1 values (99, "gotta log first");
+SET DEBUG_SYNC= 'reset';
+SET DEBUG_SYNC= "before_group_commit_queue WAIT_FOR trx1_go";
+ROLLBACK;
+connection default;
+insert into t1 values (99, "second best in binlog");
+select * from t1;
+a	b
+99	second best in binlog
+SET DEBUG_SYNC= "now SIGNAL trx1_go";
+# Prove the logging order is Trx1, Trx2
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Annotate_rows	#	#	insert into t_x values (1)
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t_x)
+master-bin.000001	#	Write_rows_v1	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Query	#	#	use `test`; insert into t1 values (99, "second best in binlog")
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Annotate_rows	#	#	insert into t1 values (99, "gotta log first")
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000001	#	Write_rows_v1	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Query	#	#	ROLLBACK
+drop table t_x, t1;
+disconnect trx1_rollback;
+# end of the tests

mysql-test/suite/binlog/t/binlog_mdev37541.test

@@ -0,0 +1,36 @@
+--source include/have_innodb.inc
+--source include/have_binlog_format_mixed.inc
+
+create table t1 (a int primary key, b text) engine=innodb;
+
+connect(trx1_rollback,localhost,root,,);
+CREATE TABLE t_x (a int) engine=MEMORY;
+
+--let $master_file= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $binlog_start=query_get_value(SHOW MASTER STATUS, Position, 1)
+SET binlog_format=row;
+CREATE TEMPORARY TABLE tt_tmp ( id INT ) ENGINE = Memory;
+BEGIN;
+  insert into t_x values (1);
+  drop temporary table tt_tmp;
+  insert into t1 values (99, "gotta log first");
+
+SET DEBUG_SYNC= 'reset';
+SET DEBUG_SYNC= "before_group_commit_queue WAIT_FOR trx1_go";
+
+--send ROLLBACK
+
+# trx2_commit
+connection default;
+insert into t1 values (99, "second best in binlog");
+select * from t1;
+
+SET DEBUG_SYNC= "now SIGNAL trx1_go";
+
+--echo # Prove the logging order is Trx1, Trx2
+--source include/show_binlog_events.inc
+
+drop table t_x, t1;
+disconnect trx1_rollback;
+
+--echo # end of the tests

sql/log.cc

@@ -8235,6 +8235,7 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry)
 bool
 MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry)
 {
+  DEBUG_SYNC(entry->thd, "before_group_commit_queue");
   int is_leader= queue_for_group_commit(entry);
 #ifdef WITH_WSREP
   /* commit order was released in queue_for_group_commit() call,