MDEV-36554 Assertion `is_wsrep() == wsrep_on(mysql_thd)' failed

           in void trx_t::commit_in_memory(const mtr_t*)

This bug fix prevents debug assertion failure when Galera feature
retry applying is enabled. This patch introduces also a general
mechanism for temporarily disabling some debug assertions in InnoDB
code.

Author: plampio

include/mysql/service_wsrep.h

@@ -8,6 +8,11 @@ enum Wsrep_service_key_type
     WSREP_SERVICE_KEY_UPDATE,
     WSREP_SERVICE_KEY_EXCLUSIVE
 };
+	
+
+/* the bits in the bitmask for disabling temporarily some asserts */
+#define  WSREP_ASSERT_INNODB_TRX   1
+
 
 #if (defined (MYSQL_DYNAMIC_PLUGIN) && defined(MYSQL_SERVICE_WSREP_DYNAMIC_INCLUDED)) || (!defined(MYSQL_DYNAMIC_PLUGIN) && defined(MYSQL_SERVICE_WSREP_STATIC_INCLUDED))
 

mysql-test/suite/galera/r/mdev-36554.result

@@ -0,0 +1,25 @@
+connection node_2;
+connection node_1;
+CALL mtr.add_suppression("Event .* Update_rows.* apply failed");
+CALL mtr.add_suppression("mariadbd: Can't find record in 't1'");
+CALL mtr.add_suppression("Failed to apply write set.*");
+CALL mtr.add_suppression("Inconsistency detected");
+connection node_1;
+CREATE TABLE t1 (f1 INTEGER);
+INSERT INTO t1 VALUES (1);
+connection node_2;
+SET SESSION wsrep_on = OFF;
+DELETE FROM t1;
+SET SESSION wsrep_on = ON;
+SET GLOBAL wsrep_applier_retry_count=1;
+connection node_1;
+UPDATE t1 SET f1 = f1 + 1;
+connection node_2;
+Shutting down server ...
+SET wsrep_on=OFF;
+Restarting server ...
+connection node_1;
+SET wsrep_sync_wait=0;
+connection node_1;
+SET GLOBAL wsrep_applier_retry_count = 0;
+DROP TABLE t1;

mysql-test/suite/galera/t/mdev-36554.test

@@ -0,0 +1,45 @@
+#
+# Test for MDEV-36554
+#
+
+--source include/galera_cluster.inc
+
+CALL mtr.add_suppression("Event .* Update_rows.* apply failed");
+CALL mtr.add_suppression("mariadbd: Can't find record in 't1'");
+CALL mtr.add_suppression("Failed to apply write set.*");
+CALL mtr.add_suppression("Inconsistency detected");
+
+--connection node_1
+CREATE TABLE t1 (f1 INTEGER);
+INSERT INTO t1 VALUES (1);
+
+--connection node_2
+SET SESSION wsrep_on = OFF;
+DELETE FROM t1;
+SET SESSION wsrep_on = ON;
+SET GLOBAL wsrep_applier_retry_count=1;
+
+--connection node_1
+UPDATE t1 SET f1 = f1 + 1;
+
+# restart node 2
+--connection node_2
+--echo Shutting down server ...
+SET wsrep_on=OFF;
+--source include/shutdown_mysqld.inc
+--source include/wait_until_disconnected.inc
+--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+--echo Restarting server ...
+--source include/start_mysqld.inc
+
+# wait till node 2 is back in the cluster
+--connection node_1
+SET wsrep_sync_wait=0;
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM performance_schema.global_status WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+# cleanup
+--connection node_1
+SET GLOBAL wsrep_applier_retry_count = 0;
+DROP TABLE t1;
+

sql/sql_class.cc

@@ -762,6 +762,7 @@ THD::THD(my_thread_id id, bool is_wsrep_applier)
    wsrep_affected_rows(0),
    wsrep_has_ignored_error(false),
    wsrep_was_on(false),
+   wsrep_assert_bitmask(0),
    wsrep_ignore_table(false),
    wsrep_aborter(0),
    wsrep_delayed_BF_abort(false),
@@ -8659,3 +8660,20 @@ LEX_CSTRING make_string(THD *thd, const char *start_ptr,
   size_t length= end_ptr - start_ptr;
   return {strmake_root(thd->mem_root, start_ptr, length), length};
 }
+
+
+#ifdef WITH_WSREP
+void wsrep_set_assert(THD *thd, uint64 assert, bool is_disabled)
+{
+  if (is_disabled) {
+    thd->wsrep_assert_bitmask |= assert;
+  } else {
+    thd->wsrep_assert_bitmask &= ~assert;
+  }
+}
+
+bool wsrep_get_assert(THD *thd, uint64 assert)
+{
+  return (thd->wsrep_assert_bitmask & assert ? true: false);
+}
+#endif /* WITH_WSREP */

sql/sql_class.h

@@ -5682,6 +5682,7 @@ class THD: public THD_count, /* this must be first */
   bool                      wsrep_has_ignored_error;
   /* true if wsrep_on was ON in last wsrep_on_update */
   bool                      wsrep_was_on;
+  uint64                    wsrep_assert_bitmask;
 
   /*
     When enabled, do not replicate/binlog updates from the current table that's
@@ -5720,7 +5721,6 @@ class THD: public THD_count, /* this must be first */
     return m_wsrep_client_state.transaction().id().get();
   }
 
-
   /*
     Set next trx id
    */
@@ -8522,5 +8522,11 @@ LEX_CSTRING make_string(THD *thd, const char *start_ptr,
 
 #include "deprecation.h"
 
+
+#ifdef WITH_WSREP
+extern void wsrep_set_assert(THD* thd, uint64 assert, bool is_disabled);
+extern bool wsrep_get_assert(THD* thd, uint64 assert);
+#endif
+
 #endif /* MYSQL_SERVER */
 #endif /* SQL_CLASS_INCLUDED */

sql/wsrep_applier.cc

@@ -292,11 +292,14 @@ int wsrep_apply_events(THD*                       thd,
 
     /* rollback to savepoint without telling Wsrep-lib */
     thd->variables.wsrep_on = false;
+    wsrep_set_assert(thd, true, WSREP_ASSERT_INNODB_TRX);
     if (FALSE != trans_rollback_to_savepoint(thd, savepoint)) {
       thd->variables.wsrep_on = true;
+      wsrep_set_assert(thd, false, WSREP_ASSERT_INNODB_TRX);
       break;
     }
     thd->variables.wsrep_on = true;
+    wsrep_set_assert(thd, false, WSREP_ASSERT_INNODB_TRX);
 
     /* reset THD object for retry */
     thd->clear_error();

storage/innobase/trx/trx0trx.cc

@@ -1382,6 +1382,10 @@ ATTRIBUTE_NOINLINE static void trx_commit_cleanup(trx_undo_t *&undo)
   undo= nullptr;
 }
 
+
+extern bool wsrep_get_assert(THD *thd, uint64 assert);
+
+
 TRANSACTIONAL_INLINE inline void trx_t::commit_in_memory(const mtr_t *mtr)
 {
   /* We already detached from rseg in write_serialisation_history() */
@@ -1501,7 +1505,9 @@ TRANSACTIONAL_INLINE inline void trx_t::commit_in_memory(const mtr_t *mtr)
     trx_finalize_for_fts(this, undo_no != 0);
 
 #ifdef WITH_WSREP
-  ut_ad(is_wsrep() == wsrep_on(mysql_thd));
+  if (!mysql_thd || !wsrep_get_assert(mysql_thd, WSREP_ASSERT_INNODB_TRX)) {
+    ut_ad(is_wsrep() == wsrep_on(mysql_thd));
+  }
 
   /* Serialization history has been written and the transaction is
   committed in memory, which makes this commit ordered. Release commit