MDEV-33175 applier FK check failure retrying

include/mysql/service_wsrep.h

@@ -89,6 +89,7 @@ extern struct wsrep_service_st {
   void                        (*wsrep_thd_kill_LOCK_func)(const MYSQL_THD thd);
   void                        (*wsrep_thd_kill_UNLOCK_func)(const MYSQL_THD thd);
   void                        (*wsrep_thd_set_wsrep_PA_unsafe_func)(MYSQL_THD thd);
+  uint                        (*wsrep_retry_FK_failure)();
 } *wsrep_service;
 
 #define MYSQL_SERVICE_WSREP_INCLUDED
@@ -133,8 +134,8 @@ extern struct wsrep_service_st {
 #define wsrep_thd_is_applying(T) wsrep_service->wsrep_thd_is_applying_func(T)
 #define wsrep_report_bf_lock_wait(T,I) wsrep_service->wsrep_report_bf_lock_wait(T,I)
 #define wsrep_thd_set_PA_unsafe(T) wsrep_service->wsrep_thd_set_PA_unsafe_func(T)
+#define wsrep_retry_FK_failure() wsrep_service->wsrep_retry_FK_failure()
 #else
-
 #define MYSQL_SERVICE_WSREP_STATIC_INCLUDED
 extern ulong   wsrep_debug;
 extern my_bool wsrep_log_conflicts;
@@ -232,5 +233,6 @@ extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
                                           unsigned long long trx_id);
 /* declare parallel applying unsafety for the THD */
 extern "C" void wsrep_thd_set_PA_unsafe(MYSQL_THD thd);
+extern "C" uint wsrep_retry_FK_failure();
 #endif
 #endif /* MYSQL_SERVICE_WSREP_INCLUDED */

mysql-test/suite/galera/r/galera_FK_failure_retry.result

@@ -0,0 +1,46 @@
+connection node_2;
+connection node_1;
+CREATE TABLE parent (
+id INT PRIMARY KEY,
+j INT
+) ENGINE=InnoDB;
+CREATE TABLE child (
+id INT PRIMARY KEY AUTO_INCREMENT,
+parent_id INT,
+KEY (parent_id),
+CONSTRAINT FOREIGN KEY (parent_id) REFERENCES parent(id)
+) ENGINE=InnoDB;
+connection node_1;
+INSERT INTO parent VALUES (1,1);
+connection node_2;
+SET GLOBAL debug_dbug ='+d,wsrep_force_FK_check_fail';
+SET GLOBAL wsrep_slave_FK_checks = ON;
+SET GLOBAL wsrep_applier_FK_failure_retries = 5;
+SET wsrep_sync_wait = 15;
+connection node_1;
+INSERT INTO child VALUES (1,1);
+connection node_2;
+SELECT * from child;
+id	parent_id
+1	1
+include/assert_grep.inc [warning for FK constraint failure exists]
+SET GLOBAL wsrep_applier_FK_failure_retries = 1;
+connection node_1;
+INSERT INTO child VALUES (2,1);
+SELECT * from child;
+id	parent_id
+1	1
+2	1
+connection node_2;
+select * from parent;
+id	j
+1	1
+SET GLOBAL debug_dbug = NULL;
+connection node_2;
+SET GLOBAL wsrep_slave_FK_checks = DEFAULT;
+SET GLOBAL wsrep_applier_FK_failure_retries = DEFAULT;
+include/assert_grep.inc [warning for FK constraint failure missing]
+CALL mtr.add_suppression("final FK constraint check failed for.*");
+connection node_1;
+DROP TABLE child;
+DROP TABLE parent;

mysql-test/suite/galera/r/galera_defaults.result

@@ -3,7 +3,7 @@ connection node_1;
 # Correct Galera library found
 SELECT COUNT(*) `expect 49` FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%';
 expect 49
-49
+50
 SELECT VARIABLE_NAME, VARIABLE_VALUE
 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
@@ -20,6 +20,7 @@ AND VARIABLE_NAME NOT IN (
 )
 ORDER BY VARIABLE_NAME;
 VARIABLE_NAME	VARIABLE_VALUE
+WSREP_APPLIER_FK_FAILURE_RETRIES	1
 WSREP_AUTO_INCREMENT_CONTROL	ON
 WSREP_CAUSAL_READS	ON
 WSREP_CERTIFICATION_RULES	strict

mysql-test/suite/galera/t/galera_FK_failure_retry.test

@@ -0,0 +1,77 @@
+--source include/galera_cluster.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/galera_have_debug_sync.inc
+
+CREATE TABLE parent (
+	id INT PRIMARY KEY,
+	j INT
+) ENGINE=InnoDB;
+
+CREATE TABLE child (
+	id INT PRIMARY KEY AUTO_INCREMENT,
+	parent_id INT,
+	KEY (parent_id),
+	CONSTRAINT FOREIGN KEY (parent_id) REFERENCES parent(id)
+) ENGINE=InnoDB;
+
+--connection node_1
+INSERT INTO parent VALUES (1,1);
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 from parent;
+--source include/wait_condition.inc
+SET GLOBAL debug_dbug ='+d,wsrep_force_FK_check_fail';
+SET GLOBAL wsrep_slave_FK_checks = ON;
+SET GLOBAL wsrep_applier_FK_failure_retries = 5;
+
+#  make sure sync wait happens in this connection
+SET wsrep_sync_wait = 15;
+
+--connection node_1
+INSERT INTO child VALUES (1,1);
+
+--connection node_2
+# Ensure that write succeeded in both nodes
+SELECT * from child;
+
+--let $assert_select = final FK constraint check failed
+--let $assert_count = 0
+--let $assert_text = warning for FK constraint failure exists
+--let $assert_only_after = CURRENT_TEST
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
+--source include/assert_grep.inc
+
+# now setting retries low so that FK failure happens
+SET GLOBAL wsrep_applier_FK_failure_retries = 1;
+
+--connection node_1
+INSERT INTO child VALUES (2,1);
+
+SELECT * from child;
+
+--connection node_2
+# must be careful to disable wsrep_force_FK_check_fail variable
+# as it could happen concurrently with applier execution
+# therefore making a non-related select which is waiting for sync wait
+#
+select * from parent;
+SET GLOBAL debug_dbug = NULL;
+
+--connection node_2
+SET GLOBAL wsrep_slave_FK_checks = DEFAULT;
+SET GLOBAL wsrep_applier_FK_failure_retries = DEFAULT;
+
+--let $assert_select = final FK constraint check failed
+--let $assert_count = 1
+--let $assert_text = warning for FK constraint failure missing
+--let $assert_only_after = CURRENT_TEST
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
+--source include/assert_grep.inc
+
+
+CALL mtr.add_suppression("final FK constraint check failed for.*");
+
+--connection node_1
+DROP TABLE child;
+DROP TABLE parent;

sql/sql_plugin_services.inl

@@ -177,7 +177,8 @@ static struct wsrep_service_st wsrep_handler = {
   wsrep_report_bf_lock_wait,
   wsrep_thd_kill_LOCK,
   wsrep_thd_kill_UNLOCK,
-  wsrep_thd_set_PA_unsafe
+  wsrep_thd_set_PA_unsafe,
+  wsrep_retry_FK_failure
 };
 
 static struct thd_specifics_service_st thd_specifics_handler=

sql/sys_vars.cc

@@ -5550,6 +5550,12 @@ static Sys_var_ulong Sys_wsrep_retry_autocommit(
        SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1));
 
+static Sys_var_uint Sys_wsrep_applier_FK_failure_retries(
+      "wsrep_applier_FK_failure_retries", "Max number of times to retry "
+      "FK constraint check failure in applying",
+       GLOBAL_VAR(wsrep_applier_FK_failure_retries), CMD_LINE(OPT_ARG),
+       VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1));
+
 static bool update_wsrep_auto_increment_control (sys_var *self, THD *thd, enum_var_type type)
 {
   if (wsrep_auto_increment_control)

sql/wsrep_mysqld.cc

@@ -97,6 +97,7 @@ my_bool wsrep_log_conflicts;
 my_bool wsrep_load_data_splitting= 0;           // Commit load data every 10K intervals
 my_bool wsrep_slave_UK_checks;                  // Slave thread does UK checks
 my_bool wsrep_slave_FK_checks;                  // Slave thread does FK checks
+uint wsrep_applier_FK_failure_retries;          // how many times FK failure to retry in applying
 my_bool wsrep_restart_slave;                    // Should mysql slave thread be
                                                 // restarted, when node joins back?
 my_bool wsrep_desync;                           // De(re)synchronize the node from the
@@ -2878,6 +2879,11 @@ int wsrep_thd_retry_counter(const THD *thd)
   return thd->wsrep_retry_counter;
 }
 
+uint wsrep_retry_FK_failure()
+{
+  return wsrep_applier_FK_failure_retries;
+}
+
 extern  bool wsrep_thd_ignore_table(THD *thd)
 {
   return thd->wsrep_ignore_table;

sql/wsrep_mysqld.h

@@ -90,6 +90,7 @@ extern bool        wsrep_gtid_mode;
 extern uint32      wsrep_gtid_domain_id;
 extern std::atomic <bool > wsrep_thread_create_failed;
 
+extern uint               wsrep_applier_FK_failure_retries;
 enum enum_wsrep_reject_types {
   WSREP_REJECT_NONE,    /* nothing rejected */
   WSREP_REJECT_ALL,     /* reject all queries, with UNKNOWN_COMMAND error */

storage/innobase/handler/ha_innodb.cc

@@ -2743,6 +2743,7 @@ innobase_trx_init(
 		thd, OPTION_RELAXED_UNIQUE_CHECKS);
 #ifdef WITH_WSREP
 	trx->wsrep = wsrep_on(thd);
+        trx->wsrep_is_BF = wsrep_thd_is_BF(thd, false);
 #endif
 
 	DBUG_VOID_RETURN;

storage/innobase/include/trx0trx.h

@@ -737,6 +737,11 @@ struct trx_t {
 	/** whether wsrep_on(mysql_thd) held at the start of transaction */
 	bool		wsrep;
 	bool is_wsrep() const { return UNIV_UNLIKELY(wsrep); }
+	/* Boolean denoting whether to trx belongs to BF thread. This is
+	   initialized at the beginning of the transaction, and the
+	   value stays the same over whole transaction lifetime. */
+	bool wsrep_is_BF;
+
 	/** true, if BF thread is performing unique secondary index scanning */
 	bool wsrep_UK_scan;
 	bool is_wsrep_UK_scan() const { return UNIV_UNLIKELY(wsrep_UK_scan); }

storage/innobase/row/row0ins.cc

@@ -3221,6 +3221,32 @@ row_ins_sec_index_entry_low(
 	mtr_commit(&mtr);
 	DBUG_RETURN(err);
 }
+#ifdef WITH_WSREP
+static uint wsrep_retries(trx_t *trx, uint err, uint retries) {
+
+  if (trx->wsrep_is_BF && err != DB_SUCCESS && err != DB_LOCK_WAIT) {
+
+    ulint trx_start, trx_end= ULINT_UNDEFINED;
+
+    if (retries > 0) {
+      retries--;
+      WSREP_DEBUG("FK constraint check retry for %d retries %d",
+                  err, retries);
+
+      if (wsrep_debug)
+        srv_printf_innodb_monitor(stderr, true, &trx_start, &trx_end);
+
+      /* force warning messages, if final retry did not help */
+      if (retries == 0) {
+        WSREP_WARN("final FK constraint check failed for %d ", err);
+        srv_printf_innodb_monitor(stderr, true, &trx_start, &trx_end);
+      }
+    }
+  } else retries = 0;
+
+  return retries;
+}
+#endif /* WITH_WSREP */
 
 /***************************************************************//**
 Inserts an entry into a clustered index. Tries first optimistic,
@@ -3240,10 +3266,28 @@ row_ins_clust_index_entry(
 	ulint	n_uniq;
 
 	DBUG_ENTER("row_ins_clust_index_entry");
+#ifdef WITH_WSREP
+	trx_t *trx= thr_get_trx(thr);
+	uint wsrep_FK_retries= (trx->wsrep_is_BF) ? wsrep_retry_FK_failure() : 0;
+	/* This counter is for galera_FK_failure_retry test,
+	DBUG_EXECUTE_IF("wsrep_force_FK_check_fail"...) below. */
+	ut_d(uint fail_count=4);
+#endif /* WITH_WSREP */
 
 	if (!index->table->foreign_set.empty()) {
+#ifdef WITH_WSREP
+		do {
+#endif /* WITH_WSREP */
 		err = row_ins_check_foreign_constraints(
 			index->table, index, true, entry, thr);
+#ifdef WITH_WSREP
+		DBUG_EXECUTE_IF("wsrep_force_FK_check_fail",
+			if (--fail_count) { err= DB_FAIL; } );
+		wsrep_FK_retries= wsrep_retries(trx, err, wsrep_FK_retries);
+		} while(wsrep_FK_retries);
+/* don't let the injected error to propagate and crash this node */
+DBUG_EXECUTE_IF("wsrep_force_FK_check_fail", err= DB_SUCCESS; );
+#endif /* WITH_WSREP */
 		if (err != DB_SUCCESS) {
 
 			DBUG_RETURN(err);
@@ -3335,10 +3379,23 @@ row_ins_sec_index_entry(
 	DBUG_EXECUTE_IF("row_ins_sec_index_entry_timeout", {
 			DBUG_SET("-d,row_ins_sec_index_entry_timeout");
 			return(DB_LOCK_WAIT);});
+#ifdef WITH_WSREP
+	trx_t *trx= thr_get_trx(thr);
+	uint wsrep_FK_retries= (trx->wsrep_is_BF) ? wsrep_retry_FK_failure() : 0;
+#endif /* WITH_WSREP */
 
 	if (check_foreign && !index->table->foreign_set.empty()) {
+#ifdef WITH_WSREP
+		do {
+#endif /* WITH_WSREP */
 		err = row_ins_check_foreign_constraints(index->table, index,
 							false, entry, thr);
+#ifdef WITH_WSREP
+		wsrep_FK_retries= wsrep_retries(trx, err, wsrep_FK_retries);
+		}
+		while(wsrep_FK_retries);
+#endif /* WITH_WSREP */
+
 		if (err != DB_SUCCESS) {
 
 			return(err);