From 2cc0cfe5db936e06b25c4c6ddcc0158d44a2e75e Mon Sep 17 00:00:00 2001 From: Alessandro Vetere Date: Tue, 5 May 2026 18:16:16 +0200 Subject: [PATCH] MDEV-38814 Reduce pessimistic update fallbacks A high rate of index lock SX-to-X upgrades was traced to two patterns in btr_cur_*_update() that turn benign UPDATEs into pessimistic fallbacks: 1. btr_cur_optimistic_update() returns DB_UNDERFLOW whenever the page would drop below BTR_CUR_PAGE_COMPRESS_LIMIT after the delete-then-insert, even when the record itself is growing. On a freshly split page this re-triggers a merge the moment the next update lands, defeating the split that just happened. 2. btr_cur_pessimistic_update() handles the DB_OVERFLOW fallback by calling btr_cur_insert_if_possible() unconditionally. When the uncompressed page cannot satisfy BTR_CUR_PAGE_REORGANIZE_LIMIT after a reorganize, that retry fails too and the same page churns through pessimistic_update without ever splitting. Introduce a system variable innodb_reduce_pessimistic_update_fallbacks (BOOL, default OFF) that gates both fixes: * btr_cur_optimistic_update() returns DB_UNDERFLOW only when the new record is strictly smaller than the old one, so growing updates do not trip the merge path. * btr_cur_pessimistic_update() skips btr_cur_insert_if_possible() and falls through to a page split when the optimistic error was DB_OVERFLOW, the page is uncompressed and still holds at least one record after the in-place delete (so the forced split produces 1+1, not 0+1), the record is growing, and a reorganize would not free BTR_CUR_PAGE_REORGANIZE_LIMIT bytes. Same-size updates take the optimistic path under both fixes (no merge, no split), matching the intent that only size-changing updates contribute to space pressure. The trade-off is that an opportunistic merge that previously triggered on any update to a sparse page now requires an actual shrink to fire. To make the impact measurable, expose seven debug-only atomic counters via SHOW GLOBAL STATUS: Innodb_btr_cur_n_index_lock_upgrades Innodb_btr_cur_pessimistic_insert_calls Innodb_btr_cur_pessimistic_update_calls Innodb_btr_cur_pessimistic_delete_calls Innodb_btr_cur_pessimistic_update_optim_err_underflows Innodb_btr_cur_pessimistic_update_optim_err_overflows Innodb_mtr_n_index_x_lock_calls A new test, innodb.index_lock_upgrade, drives a 1000-row INSERT / UPDATE / DELETE workload on a 4K-page table across three shapes (PK only; PK + secondary index on a DATETIME column; same with ROW_FORMAT=COMPRESSED, KEY_BLOCK_SIZE=2). It snapshots all seven counters plus the index_page_* INNODB_METRICS between phases. The test runs in two combinations (off and on) to lock in counter deltas for both the legacy and the optimized paths; the compressed table also covers the deliberate scope limitation that the pessimistic-side branch is gated to uncompressed pages while the optimistic-side change applies uniformly. --- .../innodb/r/index_lock_upgrade,on.rdiff | 242 ++++++ .../suite/innodb/r/index_lock_upgrade.result | 697 ++++++++++++++++++ .../innodb/r/innodb_status_variables.result | 9 +- .../innodb/t/index_lock_upgrade.combinations | 36 + .../suite/innodb/t/index_lock_upgrade.test | 212 ++++++ .../innodb/t/innodb_status_variables.test | 9 +- .../suite/sys_vars/r/sysvars_innodb.result | 12 + storage/innobase/btr/btr0cur.cc | 72 +- storage/innobase/handler/ha_innodb.cc | 62 ++ storage/innobase/include/btr0cur.h | 15 + storage/innobase/include/mtr0mtr.h | 9 +- storage/innobase/mtr/mtr0mtr.cc | 5 + 12 files changed, 1372 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/innodb/r/index_lock_upgrade,on.rdiff create mode 100644 mysql-test/suite/innodb/r/index_lock_upgrade.result create mode 100644 mysql-test/suite/innodb/t/index_lock_upgrade.combinations create mode 100644 mysql-test/suite/innodb/t/index_lock_upgrade.test diff --git a/mysql-test/suite/innodb/r/index_lock_upgrade,on.rdiff b/mysql-test/suite/innodb/r/index_lock_upgrade,on.rdiff new file mode 100644 index 0000000000000..46d40b14d9da0 --- /dev/null +++ b/mysql-test/suite/innodb/r/index_lock_upgrade,on.rdiff @@ -0,0 +1,242 @@ +--- index_lock_upgrade.result ++++ index_lock_upgrade,on.reject +@@ -52,27 +52,27 @@ + CALL show_index_stats(CONCAT('t1', '_update_null_to_notnull')); + + n_index_lock_upgrades_t1_update_null_to_notnull +-414 ++6 + pessimistic_insert_calls_t1_update_null_to_notnull +-5 ++6 + pessimistic_delete_calls_t1_update_null_to_notnull + 0 + pessimistic_update_calls_t1_update_null_to_notnull +-414 ++6 + pessimistic_update_optim_err_underflows_t1_update_null_to_notnull +-300 ++0 + pessimistic_update_optim_err_overflows_t1_update_null_to_notnull +-114 ++6 + index_x_lock_calls_t1_update_null_to_notnull + 0 + index_page_splits_t1_update_null_to_notnull +-5 ++6 + index_page_merge_attempts_t1_update_null_to_notnull +-300 ++0 + index_page_merge_successful_t1_update_null_to_notnull + 0 + index_page_reorg_successful_t1_update_null_to_notnull +-98 ++40 + index_page_discards_t1_update_null_to_notnull + 0 + # Update all rows: 2025 to 2026 (inplace update) +@@ -110,27 +110,27 @@ + CALL show_index_stats(CONCAT('t1', '_update_notnull_to_null')); + + n_index_lock_upgrades_t1_update_notnull_to_null +-380 ++422 + pessimistic_insert_calls_t1_update_notnull_to_null + 0 + pessimistic_delete_calls_t1_update_notnull_to_null +-5 ++6 + pessimistic_update_calls_t1_update_notnull_to_null +-380 ++422 + pessimistic_update_optim_err_underflows_t1_update_notnull_to_null +-311 ++342 + pessimistic_update_optim_err_overflows_t1_update_notnull_to_null +-69 ++80 + index_x_lock_calls_t1_update_notnull_to_null + 0 + index_page_splits_t1_update_notnull_to_null + 0 + index_page_merge_attempts_t1_update_notnull_to_null +-311 ++342 + index_page_merge_successful_t1_update_notnull_to_null +-5 ++6 + index_page_reorg_successful_t1_update_notnull_to_null +-5 ++6 + index_page_discards_t1_update_notnull_to_null + 0 + # Delete upper half of rows +@@ -143,11 +143,11 @@ + CALL show_index_stats(CONCAT('t1', '_delete_upper_half')); + + n_index_lock_upgrades_t1_delete_upper_half +-183 ++186 + pessimistic_insert_calls_t1_delete_upper_half + 0 + pessimistic_delete_calls_t1_delete_upper_half +-186 ++189 + pessimistic_update_calls_t1_delete_upper_half + 0 + pessimistic_update_optim_err_underflows_t1_delete_upper_half +@@ -159,7 +159,7 @@ + index_page_splits_t1_delete_upper_half + 0 + index_page_merge_attempts_t1_delete_upper_half +-183 ++186 + index_page_merge_successful_t1_delete_upper_half + 3 + index_page_reorg_successful_t1_delete_upper_half +@@ -283,27 +283,27 @@ + CALL show_index_stats(CONCAT('t2', '_update_null_to_notnull')); + + n_index_lock_upgrades_t2_update_null_to_notnull +-1016 ++608 + pessimistic_insert_calls_t2_update_null_to_notnull +-9 ++10 + pessimistic_delete_calls_t2_update_null_to_notnull + 601 + pessimistic_update_calls_t2_update_null_to_notnull +-414 ++6 + pessimistic_update_optim_err_underflows_t2_update_null_to_notnull +-300 ++0 + pessimistic_update_optim_err_overflows_t2_update_null_to_notnull +-114 ++6 + index_x_lock_calls_t2_update_null_to_notnull + 0 + index_page_splits_t2_update_null_to_notnull +-9 ++10 + index_page_merge_attempts_t2_update_null_to_notnull +-895 ++595 + index_page_merge_successful_t2_update_null_to_notnull + 0 + index_page_reorg_successful_t2_update_null_to_notnull +-98 ++40 + index_page_discards_t2_update_null_to_notnull + 3 + # Update all rows: 2025 to 2026 (inplace update) +@@ -341,27 +341,27 @@ + CALL show_index_stats(CONCAT('t2', '_update_notnull_to_null')); + + n_index_lock_upgrades_t2_update_notnull_to_null +-583 ++625 + pessimistic_insert_calls_t2_update_notnull_to_null + 3 + pessimistic_delete_calls_t2_update_notnull_to_null +-209 ++210 + pessimistic_update_calls_t2_update_notnull_to_null +-380 ++422 + pessimistic_update_optim_err_underflows_t2_update_notnull_to_null +-311 ++342 + pessimistic_update_optim_err_overflows_t2_update_notnull_to_null +-69 ++80 + index_x_lock_calls_t2_update_notnull_to_null + 0 + index_page_splits_t2_update_notnull_to_null + 3 + index_page_merge_attempts_t2_update_notnull_to_null +-511 ++542 + index_page_merge_successful_t2_update_notnull_to_null +-9 +-index_page_reorg_successful_t2_update_notnull_to_null + 10 ++index_page_reorg_successful_t2_update_notnull_to_null ++11 + index_page_discards_t2_update_notnull_to_null + 0 + # Delete upper half of rows +@@ -374,11 +374,11 @@ + CALL show_index_stats(CONCAT('t2', '_delete_upper_half')); + + n_index_lock_upgrades_t2_delete_upper_half +-340 ++343 + pessimistic_insert_calls_t2_delete_upper_half + 0 + pessimistic_delete_calls_t2_delete_upper_half +-344 ++347 + pessimistic_update_calls_t2_delete_upper_half + 0 + pessimistic_update_optim_err_underflows_t2_delete_upper_half +@@ -390,7 +390,7 @@ + index_page_splits_t2_delete_upper_half + 0 + index_page_merge_attempts_t2_delete_upper_half +-340 ++343 + index_page_merge_successful_t2_delete_upper_half + 4 + index_page_reorg_successful_t2_delete_upper_half +@@ -514,15 +514,15 @@ + CALL show_index_stats(CONCAT('t3', '_update_null_to_notnull')); + + n_index_lock_upgrades_t3_update_null_to_notnull +-1602 ++610 + pessimistic_insert_calls_t3_update_null_to_notnull + 12 + pessimistic_delete_calls_t3_update_null_to_notnull +-602 ++601 + pessimistic_update_calls_t3_update_null_to_notnull +-1000 ++8 + pessimistic_update_optim_err_underflows_t3_update_null_to_notnull +-992 ++0 + pessimistic_update_optim_err_overflows_t3_update_null_to_notnull + 0 + index_x_lock_calls_t3_update_null_to_notnull +@@ -530,9 +530,9 @@ + index_page_splits_t3_update_null_to_notnull + 12 + index_page_merge_attempts_t3_update_null_to_notnull +-1587 ++595 + index_page_merge_successful_t3_update_null_to_notnull +-1 ++0 + index_page_reorg_successful_t3_update_null_to_notnull + 0 + index_page_discards_t3_update_null_to_notnull +@@ -576,7 +576,7 @@ + pessimistic_insert_calls_t3_update_notnull_to_null + 3 + pessimistic_delete_calls_t3_update_notnull_to_null +-212 ++213 + pessimistic_update_calls_t3_update_notnull_to_null + 1000 + pessimistic_update_optim_err_underflows_t3_update_notnull_to_null +@@ -590,7 +590,7 @@ + index_page_merge_attempts_t3_update_notnull_to_null + 1200 + index_page_merge_successful_t3_update_notnull_to_null +-12 ++13 + index_page_reorg_successful_t3_update_notnull_to_null + 0 + index_page_discards_t3_update_notnull_to_null diff --git a/mysql-test/suite/innodb/r/index_lock_upgrade.result b/mysql-test/suite/innodb/r/index_lock_upgrade.result new file mode 100644 index 0000000000000..f857af1a083ac --- /dev/null +++ b/mysql-test/suite/innodb/r/index_lock_upgrade.result @@ -0,0 +1,697 @@ +# +# MDEV-38814: High rate of index_lock_upgrades due to +# btr_cur_need_opposite_intention() mostly returning true +# +# Test with table t1 (primary key only) +CREATE TABLE t1 ( +id INT NOT NULL, +dt DATETIME NULL, +PRIMARY KEY (id) +) ENGINE=InnoDB ; +# Insert rows with NULL DATETIME +InnoDB 0 transactions not purged +CALL show_index_stats(''); +INSERT INTO t1 (id, dt) SELECT seq, NULL FROM seq_1_to_1000; +# Rows inserted with NULL DATETIME +SELECT COUNT(*) AS rows_inserted FROM t1; +rows_inserted +1000 +SELECT COUNT(DISTINCT dt) AS distinct_timestamps FROM t1; +distinct_timestamps +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_insert')); + +n_index_lock_upgrades_t1_insert +5 +pessimistic_insert_calls_t1_insert +6 +pessimistic_delete_calls_t1_insert +0 +pessimistic_update_calls_t1_insert +0 +pessimistic_update_optim_err_underflows_t1_insert +0 +pessimistic_update_optim_err_overflows_t1_insert +0 +index_x_lock_calls_t1_insert +0 +index_page_splits_t1_insert +6 +index_page_merge_attempts_t1_insert +0 +index_page_merge_successful_t1_insert +0 +index_page_reorg_successful_t1_insert +0 +index_page_discards_t1_insert +0 +# Update all rows: NULL to DATETIME +UPDATE t1 SET dt= '2025-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_update_null_to_notnull')); + +n_index_lock_upgrades_t1_update_null_to_notnull +414 +pessimistic_insert_calls_t1_update_null_to_notnull +5 +pessimistic_delete_calls_t1_update_null_to_notnull +0 +pessimistic_update_calls_t1_update_null_to_notnull +414 +pessimistic_update_optim_err_underflows_t1_update_null_to_notnull +300 +pessimistic_update_optim_err_overflows_t1_update_null_to_notnull +114 +index_x_lock_calls_t1_update_null_to_notnull +0 +index_page_splits_t1_update_null_to_notnull +5 +index_page_merge_attempts_t1_update_null_to_notnull +300 +index_page_merge_successful_t1_update_null_to_notnull +0 +index_page_reorg_successful_t1_update_null_to_notnull +98 +index_page_discards_t1_update_null_to_notnull +0 +# Update all rows: 2025 to 2026 (inplace update) +UPDATE t1 SET dt= '2026-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_update_inplace')); + +n_index_lock_upgrades_t1_update_inplace +0 +pessimistic_insert_calls_t1_update_inplace +0 +pessimistic_delete_calls_t1_update_inplace +0 +pessimistic_update_calls_t1_update_inplace +0 +pessimistic_update_optim_err_underflows_t1_update_inplace +0 +pessimistic_update_optim_err_overflows_t1_update_inplace +0 +index_x_lock_calls_t1_update_inplace +0 +index_page_splits_t1_update_inplace +0 +index_page_merge_attempts_t1_update_inplace +0 +index_page_merge_successful_t1_update_inplace +0 +index_page_reorg_successful_t1_update_inplace +0 +index_page_discards_t1_update_inplace +0 +# Update all rows: DATETIME to NULL +UPDATE t1 SET dt= NULL; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_update_notnull_to_null')); + +n_index_lock_upgrades_t1_update_notnull_to_null +380 +pessimistic_insert_calls_t1_update_notnull_to_null +0 +pessimistic_delete_calls_t1_update_notnull_to_null +5 +pessimistic_update_calls_t1_update_notnull_to_null +380 +pessimistic_update_optim_err_underflows_t1_update_notnull_to_null +311 +pessimistic_update_optim_err_overflows_t1_update_notnull_to_null +69 +index_x_lock_calls_t1_update_notnull_to_null +0 +index_page_splits_t1_update_notnull_to_null +0 +index_page_merge_attempts_t1_update_notnull_to_null +311 +index_page_merge_successful_t1_update_notnull_to_null +5 +index_page_reorg_successful_t1_update_notnull_to_null +5 +index_page_discards_t1_update_notnull_to_null +0 +# Delete upper half of rows +DELETE FROM t1 WHERE id > (SELECT MAX(id) FROM t1) DIV 2; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t1; +rows_remaining +500 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_delete_upper_half')); + +n_index_lock_upgrades_t1_delete_upper_half +183 +pessimistic_insert_calls_t1_delete_upper_half +0 +pessimistic_delete_calls_t1_delete_upper_half +186 +pessimistic_update_calls_t1_delete_upper_half +0 +pessimistic_update_optim_err_underflows_t1_delete_upper_half +0 +pessimistic_update_optim_err_overflows_t1_delete_upper_half +0 +index_x_lock_calls_t1_delete_upper_half +0 +index_page_splits_t1_delete_upper_half +0 +index_page_merge_attempts_t1_delete_upper_half +183 +index_page_merge_successful_t1_delete_upper_half +3 +index_page_reorg_successful_t1_delete_upper_half +3 +index_page_discards_t1_delete_upper_half +0 +# Delete even-numbered rows +DELETE FROM t1 WHERE id % 2 = 0; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t1; +rows_remaining +250 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_delete_even')); + +n_index_lock_upgrades_t1_delete_even +13 +pessimistic_insert_calls_t1_delete_even +0 +pessimistic_delete_calls_t1_delete_even +15 +pessimistic_update_calls_t1_delete_even +0 +pessimistic_update_optim_err_underflows_t1_delete_even +0 +pessimistic_update_optim_err_overflows_t1_delete_even +0 +index_x_lock_calls_t1_delete_even +0 +index_page_splits_t1_delete_even +0 +index_page_merge_attempts_t1_delete_even +13 +index_page_merge_successful_t1_delete_even +2 +index_page_reorg_successful_t1_delete_even +2 +index_page_discards_t1_delete_even +0 +# Delete remaining rows +DELETE FROM t1; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t1; +rows_remaining +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t1', '_delete_remaining')); + +n_index_lock_upgrades_t1_delete_remaining +16 +pessimistic_insert_calls_t1_delete_remaining +0 +pessimistic_delete_calls_t1_delete_remaining +17 +pessimistic_update_calls_t1_delete_remaining +0 +pessimistic_update_optim_err_underflows_t1_delete_remaining +0 +pessimistic_update_optim_err_overflows_t1_delete_remaining +0 +index_x_lock_calls_t1_delete_remaining +0 +index_page_splits_t1_delete_remaining +0 +index_page_merge_attempts_t1_delete_remaining +16 +index_page_merge_successful_t1_delete_remaining +2 +index_page_reorg_successful_t1_delete_remaining +1 +index_page_discards_t1_delete_remaining +0 +DROP TABLE t1; +# Test with table t2 (primary key + secondary index on dt) +CREATE TABLE t2 ( +id INT NOT NULL, +dt DATETIME NULL, +PRIMARY KEY (id), KEY (dt) +) ENGINE=InnoDB ; +# Insert rows with NULL DATETIME +InnoDB 0 transactions not purged +CALL show_index_stats(''); +INSERT INTO t2 (id, dt) SELECT seq, NULL FROM seq_1_to_1000; +# Rows inserted with NULL DATETIME +SELECT COUNT(*) AS rows_inserted FROM t2; +rows_inserted +1000 +SELECT COUNT(DISTINCT dt) AS distinct_timestamps FROM t2; +distinct_timestamps +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_insert')); + +n_index_lock_upgrades_t2_insert +7 +pessimistic_insert_calls_t2_insert +9 +pessimistic_delete_calls_t2_insert +0 +pessimistic_update_calls_t2_insert +0 +pessimistic_update_optim_err_underflows_t2_insert +0 +pessimistic_update_optim_err_overflows_t2_insert +0 +index_x_lock_calls_t2_insert +0 +index_page_splits_t2_insert +9 +index_page_merge_attempts_t2_insert +0 +index_page_merge_successful_t2_insert +0 +index_page_reorg_successful_t2_insert +0 +index_page_discards_t2_insert +0 +# Update all rows: NULL to DATETIME +UPDATE t2 SET dt= '2025-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_update_null_to_notnull')); + +n_index_lock_upgrades_t2_update_null_to_notnull +1016 +pessimistic_insert_calls_t2_update_null_to_notnull +9 +pessimistic_delete_calls_t2_update_null_to_notnull +601 +pessimistic_update_calls_t2_update_null_to_notnull +414 +pessimistic_update_optim_err_underflows_t2_update_null_to_notnull +300 +pessimistic_update_optim_err_overflows_t2_update_null_to_notnull +114 +index_x_lock_calls_t2_update_null_to_notnull +0 +index_page_splits_t2_update_null_to_notnull +9 +index_page_merge_attempts_t2_update_null_to_notnull +895 +index_page_merge_successful_t2_update_null_to_notnull +0 +index_page_reorg_successful_t2_update_null_to_notnull +98 +index_page_discards_t2_update_null_to_notnull +3 +# Update all rows: 2025 to 2026 (inplace update) +UPDATE t2 SET dt= '2026-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_update_inplace')); + +n_index_lock_upgrades_t2_update_inplace +551 +pessimistic_insert_calls_t2_update_inplace +3 +pessimistic_delete_calls_t2_update_inplace +552 +pessimistic_update_calls_t2_update_inplace +0 +pessimistic_update_optim_err_underflows_t2_update_inplace +0 +pessimistic_update_optim_err_overflows_t2_update_inplace +0 +index_x_lock_calls_t2_update_inplace +0 +index_page_splits_t2_update_inplace +3 +index_page_merge_attempts_t2_update_inplace +544 +index_page_merge_successful_t2_update_inplace +0 +index_page_reorg_successful_t2_update_inplace +0 +index_page_discards_t2_update_inplace +4 +# Update all rows: DATETIME to NULL +UPDATE t2 SET dt= NULL; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_update_notnull_to_null')); + +n_index_lock_upgrades_t2_update_notnull_to_null +583 +pessimistic_insert_calls_t2_update_notnull_to_null +3 +pessimistic_delete_calls_t2_update_notnull_to_null +209 +pessimistic_update_calls_t2_update_notnull_to_null +380 +pessimistic_update_optim_err_underflows_t2_update_notnull_to_null +311 +pessimistic_update_optim_err_overflows_t2_update_notnull_to_null +69 +index_x_lock_calls_t2_update_notnull_to_null +0 +index_page_splits_t2_update_notnull_to_null +3 +index_page_merge_attempts_t2_update_notnull_to_null +511 +index_page_merge_successful_t2_update_notnull_to_null +9 +index_page_reorg_successful_t2_update_notnull_to_null +10 +index_page_discards_t2_update_notnull_to_null +0 +# Delete upper half of rows +DELETE FROM t2 WHERE id > (SELECT MAX(id) FROM t2) DIV 2; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t2; +rows_remaining +500 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_delete_upper_half')); + +n_index_lock_upgrades_t2_delete_upper_half +340 +pessimistic_insert_calls_t2_delete_upper_half +0 +pessimistic_delete_calls_t2_delete_upper_half +344 +pessimistic_update_calls_t2_delete_upper_half +0 +pessimistic_update_optim_err_underflows_t2_delete_upper_half +0 +pessimistic_update_optim_err_overflows_t2_delete_upper_half +0 +index_x_lock_calls_t2_delete_upper_half +0 +index_page_splits_t2_delete_upper_half +0 +index_page_merge_attempts_t2_delete_upper_half +340 +index_page_merge_successful_t2_delete_upper_half +4 +index_page_reorg_successful_t2_delete_upper_half +4 +index_page_discards_t2_delete_upper_half +0 +# Delete even-numbered rows +DELETE FROM t2 WHERE id % 2 = 0; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t2; +rows_remaining +250 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_delete_even')); + +n_index_lock_upgrades_t2_delete_even +15 +pessimistic_insert_calls_t2_delete_even +0 +pessimistic_delete_calls_t2_delete_even +18 +pessimistic_update_calls_t2_delete_even +0 +pessimistic_update_optim_err_underflows_t2_delete_even +0 +pessimistic_update_optim_err_overflows_t2_delete_even +0 +index_x_lock_calls_t2_delete_even +0 +index_page_splits_t2_delete_even +0 +index_page_merge_attempts_t2_delete_even +15 +index_page_merge_successful_t2_delete_even +4 +index_page_reorg_successful_t2_delete_even +3 +index_page_discards_t2_delete_even +0 +# Delete remaining rows +DELETE FROM t2; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t2; +rows_remaining +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t2', '_delete_remaining')); + +n_index_lock_upgrades_t2_delete_remaining +16 +pessimistic_insert_calls_t2_delete_remaining +0 +pessimistic_delete_calls_t2_delete_remaining +17 +pessimistic_update_calls_t2_delete_remaining +0 +pessimistic_update_optim_err_underflows_t2_delete_remaining +0 +pessimistic_update_optim_err_overflows_t2_delete_remaining +0 +index_x_lock_calls_t2_delete_remaining +0 +index_page_splits_t2_delete_remaining +0 +index_page_merge_attempts_t2_delete_remaining +16 +index_page_merge_successful_t2_delete_remaining +2 +index_page_reorg_successful_t2_delete_remaining +1 +index_page_discards_t2_delete_remaining +0 +DROP TABLE t2; +# Test with table t3 (primary key + secondary index on dt, ROW_FORMAT=COMPRESSED) +CREATE TABLE t3 ( +id INT NOT NULL, +dt DATETIME NULL, +PRIMARY KEY (id), KEY (dt) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=2; +# Insert rows with NULL DATETIME +InnoDB 0 transactions not purged +CALL show_index_stats(''); +INSERT INTO t3 (id, dt) SELECT seq, NULL FROM seq_1_to_1000; +# Rows inserted with NULL DATETIME +SELECT COUNT(*) AS rows_inserted FROM t3; +rows_inserted +1000 +SELECT COUNT(DISTINCT dt) AS distinct_timestamps FROM t3; +distinct_timestamps +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_insert')); + +n_index_lock_upgrades_t3_insert +10 +pessimistic_insert_calls_t3_insert +12 +pessimistic_delete_calls_t3_insert +0 +pessimistic_update_calls_t3_insert +0 +pessimistic_update_optim_err_underflows_t3_insert +0 +pessimistic_update_optim_err_overflows_t3_insert +0 +index_x_lock_calls_t3_insert +0 +index_page_splits_t3_insert +12 +index_page_merge_attempts_t3_insert +0 +index_page_merge_successful_t3_insert +0 +index_page_reorg_successful_t3_insert +0 +index_page_discards_t3_insert +0 +# Update all rows: NULL to DATETIME +UPDATE t3 SET dt= '2025-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_update_null_to_notnull')); + +n_index_lock_upgrades_t3_update_null_to_notnull +1602 +pessimistic_insert_calls_t3_update_null_to_notnull +12 +pessimistic_delete_calls_t3_update_null_to_notnull +602 +pessimistic_update_calls_t3_update_null_to_notnull +1000 +pessimistic_update_optim_err_underflows_t3_update_null_to_notnull +992 +pessimistic_update_optim_err_overflows_t3_update_null_to_notnull +0 +index_x_lock_calls_t3_update_null_to_notnull +0 +index_page_splits_t3_update_null_to_notnull +12 +index_page_merge_attempts_t3_update_null_to_notnull +1587 +index_page_merge_successful_t3_update_null_to_notnull +1 +index_page_reorg_successful_t3_update_null_to_notnull +0 +index_page_discards_t3_update_null_to_notnull +3 +# Update all rows: 2025 to 2026 (inplace update) +UPDATE t3 SET dt= '2026-01-01' + INTERVAL id SECOND; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_update_inplace')); + +n_index_lock_upgrades_t3_update_inplace +551 +pessimistic_insert_calls_t3_update_inplace +3 +pessimistic_delete_calls_t3_update_inplace +552 +pessimistic_update_calls_t3_update_inplace +0 +pessimistic_update_optim_err_underflows_t3_update_inplace +0 +pessimistic_update_optim_err_overflows_t3_update_inplace +0 +index_x_lock_calls_t3_update_inplace +0 +index_page_splits_t3_update_inplace +3 +index_page_merge_attempts_t3_update_inplace +544 +index_page_merge_successful_t3_update_inplace +0 +index_page_reorg_successful_t3_update_inplace +0 +index_page_discards_t3_update_inplace +4 +# Update all rows: DATETIME to NULL +UPDATE t3 SET dt= NULL; +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_update_notnull_to_null')); + +n_index_lock_upgrades_t3_update_notnull_to_null +1203 +pessimistic_insert_calls_t3_update_notnull_to_null +3 +pessimistic_delete_calls_t3_update_notnull_to_null +212 +pessimistic_update_calls_t3_update_notnull_to_null +1000 +pessimistic_update_optim_err_underflows_t3_update_notnull_to_null +1000 +pessimistic_update_optim_err_overflows_t3_update_notnull_to_null +0 +index_x_lock_calls_t3_update_notnull_to_null +0 +index_page_splits_t3_update_notnull_to_null +3 +index_page_merge_attempts_t3_update_notnull_to_null +1200 +index_page_merge_successful_t3_update_notnull_to_null +12 +index_page_reorg_successful_t3_update_notnull_to_null +0 +index_page_discards_t3_update_notnull_to_null +0 +# Delete upper half of rows +DELETE FROM t3 WHERE id > (SELECT MAX(id) FROM t3) DIV 2; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t3; +rows_remaining +500 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_delete_upper_half')); + +n_index_lock_upgrades_t3_delete_upper_half +450 +pessimistic_insert_calls_t3_delete_upper_half +0 +pessimistic_delete_calls_t3_delete_upper_half +455 +pessimistic_update_calls_t3_delete_upper_half +0 +pessimistic_update_optim_err_underflows_t3_delete_upper_half +0 +pessimistic_update_optim_err_overflows_t3_delete_upper_half +0 +index_x_lock_calls_t3_delete_upper_half +0 +index_page_splits_t3_delete_upper_half +0 +index_page_merge_attempts_t3_delete_upper_half +450 +index_page_merge_successful_t3_delete_upper_half +5 +index_page_reorg_successful_t3_delete_upper_half +0 +index_page_discards_t3_delete_upper_half +0 +# Delete even-numbered rows +DELETE FROM t3 WHERE id % 2 = 0; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t3; +rows_remaining +250 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_delete_even')); + +n_index_lock_upgrades_t3_delete_even +123 +pessimistic_insert_calls_t3_delete_even +0 +pessimistic_delete_calls_t3_delete_even +125 +pessimistic_update_calls_t3_delete_even +0 +pessimistic_update_optim_err_underflows_t3_delete_even +0 +pessimistic_update_optim_err_overflows_t3_delete_even +0 +index_x_lock_calls_t3_delete_even +0 +index_page_splits_t3_delete_even +0 +index_page_merge_attempts_t3_delete_even +123 +index_page_merge_successful_t3_delete_even +3 +index_page_reorg_successful_t3_delete_even +0 +index_page_discards_t3_delete_even +0 +# Delete remaining rows +DELETE FROM t3; +# Rows remaining +SELECT COUNT(*) AS rows_remaining FROM t3; +rows_remaining +0 +InnoDB 0 transactions not purged +CALL show_index_stats(CONCAT('t3', '_delete_remaining')); + +n_index_lock_upgrades_t3_delete_remaining +89 +pessimistic_insert_calls_t3_delete_remaining +0 +pessimistic_delete_calls_t3_delete_remaining +92 +pessimistic_update_calls_t3_delete_remaining +0 +pessimistic_update_optim_err_underflows_t3_delete_remaining +0 +pessimistic_update_optim_err_overflows_t3_delete_remaining +0 +index_x_lock_calls_t3_delete_remaining +0 +index_page_splits_t3_delete_remaining +0 +index_page_merge_attempts_t3_delete_remaining +89 +index_page_merge_successful_t3_delete_remaining +4 +index_page_reorg_successful_t3_delete_remaining +0 +index_page_discards_t3_delete_remaining +0 +DROP TABLE t3; diff --git a/mysql-test/suite/innodb/r/innodb_status_variables.result b/mysql-test/suite/innodb/r/innodb_status_variables.result index c6f4d4f27c45a..904eb94b743b7 100644 --- a/mysql-test/suite/innodb/r/innodb_status_variables.result +++ b/mysql-test/suite/innodb/r/innodb_status_variables.result @@ -3,7 +3,14 @@ WHERE variable_name LIKE 'INNODB_%' AND variable_name NOT IN ('INNODB_ADAPTIVE_HASH_HASH_SEARCHES','INNODB_ADAPTIVE_HASH_NON_HASH_SEARCHES', 'INNODB_MEM_ADAPTIVE_HASH', -'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED'); +'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED', +'INNODB_BTR_CUR_N_INDEX_LOCK_UPGRADES', +'INNODB_BTR_CUR_PESSIMISTIC_INSERT_CALLS', +'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_CALLS', +'INNODB_BTR_CUR_PESSIMISTIC_DELETE_CALLS', +'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_OPTIM_ERR_UNDERFLOWS', +'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_OPTIM_ERR_OVERFLOWS', +'INNODB_MTR_N_INDEX_X_LOCK_CALLS'); variable_name INNODB_ASYNC_READS_PENDING INNODB_ASYNC_READS_TASKS_RUNNING diff --git a/mysql-test/suite/innodb/t/index_lock_upgrade.combinations b/mysql-test/suite/innodb/t/index_lock_upgrade.combinations new file mode 100644 index 0000000000000..a40bd7c4315a6 --- /dev/null +++ b/mysql-test/suite/innodb/t/index_lock_upgrade.combinations @@ -0,0 +1,36 @@ +# MDEV-38814: exercise both states of +# innodb_reduce_pessimistic_update_fallbacks. +# +# [off]: legacy behavior. +# +# [on]: optimizations enabled; all DB_UNDERFLOW fallbacks on growing +# records are eliminated, DB_OVERFLOW fallbacks on growing records are +# forced to split when the page cannot satisfy +# BTR_CUR_PAGE_REORGANIZE_LIMIT, and the index-lock upgrade count drops +# accordingly on the same workload. +# Same-size records UPDATEs are unaffected. +# Shrinking records UPDATEs are marginally affected, because they occur +# after the record growing UPDATEs are executed, and an extra page is +# produced via split as a side-effect of the optimization, so they +# operate on a slightly different B-tree. +# +# Compressed pages (t3) get the optimistic-side change but not the +# pessimistic-side one, by design. The optimistic gate +# (page_get_data_size vs BTR_CUR_PAGE_COMPRESS_LIMIT) is an +# uncompressed-frame metric that translates cleanly to compressed pages, +# so it applies uniformly. The pessimistic gate +# (page_get_max_insert_size_after_reorganize vs +# BTR_CUR_PAGE_REORGANIZE_LIMIT) is also an uncompressed-frame metric, +# but the actual constraint on a compressed page is whether the +# compressed image fits in KEY_BLOCK_SIZE after recompress -- a +# different question. Using the uncompressed metric there would +# mispredict in both directions, so btr_cur_pessimistic_update() falls +# through to the legacy btr_cur_insert_if_possible() path, which +# already handles compressed-page complexity. The t3 rdiff therefore +# shows on/off deltas only on the optimistic-side counters. + +[off] +--loose-innodb-reduce-pessimistic-update-fallbacks=OFF + +[on] +--loose-innodb-reduce-pessimistic-update-fallbacks=ON diff --git a/mysql-test/suite/innodb/t/index_lock_upgrade.test b/mysql-test/suite/innodb/t/index_lock_upgrade.test new file mode 100644 index 0000000000000..bde08dab6ad0e --- /dev/null +++ b/mysql-test/suite/innodb/t/index_lock_upgrade.test @@ -0,0 +1,212 @@ +--source include/have_innodb.inc +--source include/have_innodb_4k.inc +--source include/have_sequence.inc +--source include/have_debug.inc +# Debug build is necessary for the Innodb_* metrics used in this test + +--echo # +--echo # MDEV-38814: High rate of index_lock_upgrades due to +--echo # btr_cur_need_opposite_intention() mostly returning true +--echo # + +--disable_query_log +SET @save_stats_persistent= @@GLOBAL.innodb_stats_persistent; +SET GLOBAL innodb_stats_persistent= 0; +# Enable index_page_* metrics +SET GLOBAL innodb_monitor_enable= 'module_index'; + +# show_index_stats(suffix) encapsulates the pessimistic-operation +# monitoring. It snapshots the relevant InnoDB global status counters +# and INNODB_METRICS entries into session variables @*_curr, shifts +# the previous snapshot into @*_prev, and -- when suffix is non-empty +# -- emits one row per counter with the delta. Pass an empty suffix +# to only establish the baseline snapshot. +DELIMITER |; +CREATE PROCEDURE show_index_stats(IN suffix VARCHAR(64)) +BEGIN + SET @ps_prev= IFNULL(@ps_curr, 0); + SET @ma_prev= IFNULL(@ma_curr, 0); + SET @ms_prev= IFNULL(@ms_curr, 0); + SET @rs_prev= IFNULL(@rs_curr, 0); + SET @pd_prev= IFNULL(@pd_curr, 0); + SET @lu_prev= IFNULL(@lu_curr, 0); + SET @pi_prev= IFNULL(@pi_curr, 0); + SET @pe_prev= IFNULL(@pe_curr, 0); + SET @pu_prev= IFNULL(@pu_curr, 0); + SET @ou_prev= IFNULL(@ou_curr, 0); + SET @oo_prev= IFNULL(@oo_curr, 0); + SET @ix_prev= IFNULL(@ix_curr, 0); + + SELECT COUNT_RESET INTO @ps_curr FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME= 'index_page_splits'; + SELECT COUNT_RESET INTO @ma_curr FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME= 'index_page_merge_attempts'; + SELECT COUNT_RESET INTO @ms_curr FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME= 'index_page_merge_successful'; + SELECT COUNT_RESET INTO @rs_curr FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME= 'index_page_reorg_successful'; + SELECT COUNT_RESET INTO @pd_curr FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME= 'index_page_discards'; + SELECT VARIABLE_VALUE INTO @lu_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_n_index_lock_upgrades'; + SELECT VARIABLE_VALUE INTO @pi_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_pessimistic_insert_calls'; + SELECT VARIABLE_VALUE INTO @pe_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_pessimistic_delete_calls'; + SELECT VARIABLE_VALUE INTO @pu_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_pessimistic_update_calls'; + SELECT VARIABLE_VALUE INTO @ou_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_pessimistic_update_optim_err_underflows'; + SELECT VARIABLE_VALUE INTO @oo_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_btr_cur_pessimistic_update_optim_err_overflows'; + SELECT VARIABLE_VALUE INTO @ix_curr FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME= 'Innodb_mtr_n_index_x_lock_calls'; + + IF suffix <> '' THEN + SELECT CONCAT('n_index_lock_upgrades_', suffix) AS '' + UNION ALL + SELECT CAST(@lu_curr - @lu_prev AS SIGNED) + UNION ALL + SELECT CONCAT('pessimistic_insert_calls_', suffix) + UNION ALL + SELECT CAST(@pi_curr - @pi_prev AS SIGNED) + UNION ALL + SELECT CONCAT('pessimistic_delete_calls_', suffix) + UNION ALL + SELECT CAST(@pe_curr - @pe_prev AS SIGNED) + UNION ALL + SELECT CONCAT('pessimistic_update_calls_', suffix) + UNION ALL + SELECT CAST(@pu_curr - @pu_prev AS SIGNED) + UNION ALL + SELECT CONCAT('pessimistic_update_optim_err_underflows_', suffix) + UNION ALL + SELECT CAST(@ou_curr - @ou_prev AS SIGNED) + UNION ALL + SELECT CONCAT('pessimistic_update_optim_err_overflows_', suffix) + UNION ALL + SELECT CAST(@oo_curr - @oo_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_x_lock_calls_', suffix) + UNION ALL + SELECT CAST(@ix_curr - @ix_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_page_splits_', suffix) + UNION ALL + SELECT CAST(@ps_curr - @ps_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_page_merge_attempts_', suffix) + UNION ALL + SELECT CAST(@ma_curr - @ma_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_page_merge_successful_', suffix) + UNION ALL + SELECT CAST(@ms_curr - @ms_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_page_reorg_successful_', suffix) + UNION ALL + SELECT CAST(@rs_curr - @rs_prev AS SIGNED) + UNION ALL + SELECT CONCAT('index_page_discards_', suffix) + UNION ALL + SELECT CAST(@pd_curr - @pd_prev AS SIGNED); + END IF; +END| +DELIMITER ;| +--enable_query_log + +# Run the same workload against three table shapes: +# t1 -- primary key only +# t2 -- primary key + secondary index on dt +# t3 -- primary key + secondary index on dt, ROW_FORMAT=COMPRESSED +# (sanity check: pessimistic-side optimization explicitly skips +# compressed pages, so on/off counter deltas should match) +--let $iter= 1 +while ($iter <= 3) +{ + --let $tab= t$iter + --let $desc= primary key only + --let $sec_key= + --let $row_format= + if ($iter == 2) + { + --let $desc= primary key + secondary index on dt + --let $sec_key= , KEY (dt) + } + if ($iter == 3) + { + --let $desc= primary key + secondary index on dt, ROW_FORMAT=COMPRESSED + --let $sec_key= , KEY (dt) + --let $row_format= ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=2 + } + --echo # Test with table $tab ($desc) + eval CREATE TABLE $tab ( + id INT NOT NULL, + dt DATETIME NULL, + PRIMARY KEY (id)$sec_key + ) ENGINE=InnoDB $row_format; + + --echo # Insert rows with NULL DATETIME + --source include/wait_all_purged.inc + CALL show_index_stats(''); + + eval INSERT INTO $tab (id, dt) SELECT seq, NULL FROM seq_1_to_1000; + + --echo # Rows inserted with NULL DATETIME + eval SELECT COUNT(*) AS rows_inserted FROM $tab; + eval SELECT COUNT(DISTINCT dt) AS distinct_timestamps FROM $tab; + + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_insert')); + + --echo # Update all rows: NULL to DATETIME + eval UPDATE $tab SET dt= '2025-01-01' + INTERVAL id SECOND; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_update_null_to_notnull')); + + --echo # Update all rows: 2025 to 2026 (inplace update) + eval UPDATE $tab SET dt= '2026-01-01' + INTERVAL id SECOND; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_update_inplace')); + + --echo # Update all rows: DATETIME to NULL + eval UPDATE $tab SET dt= NULL; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_update_notnull_to_null')); + + --echo # Delete upper half of rows + eval DELETE FROM $tab WHERE id > (SELECT MAX(id) FROM $tab) DIV 2; + --echo # Rows remaining + eval SELECT COUNT(*) AS rows_remaining FROM $tab; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_delete_upper_half')); + + --echo # Delete even-numbered rows + eval DELETE FROM $tab WHERE id % 2 = 0; + --echo # Rows remaining + eval SELECT COUNT(*) AS rows_remaining FROM $tab; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_delete_even')); + + --echo # Delete remaining rows + eval DELETE FROM $tab; + --echo # Rows remaining + eval SELECT COUNT(*) AS rows_remaining FROM $tab; + --source include/wait_all_purged.inc + eval CALL show_index_stats(CONCAT('$tab', '_delete_remaining')); + + eval DROP TABLE $tab; + + --inc $iter +} + +--disable_query_log +DROP PROCEDURE show_index_stats; +SET GLOBAL innodb_stats_persistent= @save_stats_persistent; +SET GLOBAL innodb_monitor_disable= 'module_index'; +--disable_warnings +# Idiomatic way of resetting innodb_monitor_* variables +SET GLOBAL innodb_monitor_disable= default; +SET GLOBAL innodb_monitor_enable= default; +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/innodb/t/innodb_status_variables.test b/mysql-test/suite/innodb/t/innodb_status_variables.test index 6746a94530fcc..b6583e771b206 100644 --- a/mysql-test/suite/innodb/t/innodb_status_variables.test +++ b/mysql-test/suite/innodb/t/innodb_status_variables.test @@ -4,4 +4,11 @@ WHERE variable_name LIKE 'INNODB_%' AND variable_name NOT IN ('INNODB_ADAPTIVE_HASH_HASH_SEARCHES','INNODB_ADAPTIVE_HASH_NON_HASH_SEARCHES', 'INNODB_MEM_ADAPTIVE_HASH', - 'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED'); + 'INNODB_BUFFERED_AIO_SUBMITTED','INNODB_BUFFER_POOL_PAGES_LATCHED', + 'INNODB_BTR_CUR_N_INDEX_LOCK_UPGRADES', + 'INNODB_BTR_CUR_PESSIMISTIC_INSERT_CALLS', + 'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_CALLS', + 'INNODB_BTR_CUR_PESSIMISTIC_DELETE_CALLS', + 'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_OPTIM_ERR_UNDERFLOWS', + 'INNODB_BTR_CUR_PESSIMISTIC_UPDATE_OPTIM_ERR_OVERFLOWS', + 'INNODB_MTR_N_INDEX_X_LOCK_CALLS'); diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 988eb248c0257..c084ed0515a27 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1378,6 +1378,18 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_REDUCE_PESSIMISTIC_UPDATE_FALLBACKS +SESSION_VALUE NULL +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Enable the btr_cur_*_update() optimizations when the UPDATE is increasing the size of a record: gate the DB_UNDERFLOW return of btr_cur_optimistic_update() behind actual record shrinkage so freshly split pages are not re-merged when an update is growing a record, and skip the in-place reinsert in btr_cur_pessimistic_update() when an uncompressed page cannot satisfy BTR_CUR_PAGE_REORGANIZE_LIMIT after reorganize, falling through to a page split instead. Trades some B-tree self-healing for a lower rate of pessimistic update fallbacks and the index-lock upgrades they induce +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_ROLLBACK_ON_TIMEOUT SESSION_VALUE NULL DEFAULT_VALUE OFF diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 40a7305062ad5..3fd2f25ea1d2d 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -105,8 +105,31 @@ ulint btr_cur_n_sea_old; #ifdef UNIV_DEBUG /* Flag to limit optimistic insert records */ uint btr_cur_limit_optimistic_insert_debug; +/** Number of times index lock was upgraded from SX to X */ +Atomic_counter btr_cur_n_index_lock_upgrades{0}; +/** Number of times btr_cur_pessimistic_insert() was called */ +Atomic_counter btr_cur_pessimistic_insert_calls{0}; +/** Number of times btr_cur_pessimistic_update() was called */ +Atomic_counter btr_cur_pessimistic_update_calls{0}; +/** Number of times btr_cur_pessimistic_delete() was called */ +Atomic_counter btr_cur_pessimistic_delete_calls{0}; +/** Number of times DB_UNDERFLOW was returned as optimistic update error in btr_cur_pessimistic_update() */ +Atomic_counter btr_cur_pessimistic_update_optim_err_underflows{0}; +/** Number of times DB_OVERFLOW was returned as optimistic update error in btr_cur_pessimistic_update() */ +Atomic_counter btr_cur_pessimistic_update_optim_err_overflows{0}; #endif /* UNIV_DEBUG */ +/** innodb_reduce_pessimistic_update_fallbacks: enable the MDEV-38814 +pessimistic-update optimizations. When ON, btr_cur_optimistic_update() +gates its DB_UNDERFLOW return behind actual record shrinkage (so a +freshly split page is not re-merged just because an update grew a +record), and btr_cur_pessimistic_update() skips btr_cur_insert_if_possible() +on the DB_OVERFLOW fallback for a growing record when an uncompressed +page cannot satisfy BTR_CUR_PAGE_REORGANIZE_LIMIT after a reorganize, +falling through to a page split instead (only size-growing UPDATEs pay +the cost of an early page split). */ +my_bool btr_cur_reduce_pessimistic_update_fallbacks; + /** In the optimistic insert, if the insert does not fit, but this much space can be released by page reorganize, then it is reorganized */ #define BTR_CUR_PAGE_REORGANIZE_LIMIT (srv_page_size / 32) @@ -1599,6 +1622,7 @@ ATTRIBUTE_COLD void mtr_t::index_lock_upgrade() index_lock *lock= static_cast(slot.object); lock->u_x_upgrade(SRW_LOCK_CALL); slot.type= MTR_MEMO_X_LOCK; + ut_d(++btr_cur_n_index_lock_upgrades); } /** Mark a non-leaf page "least recently used", but avoid invoking @@ -2573,6 +2597,8 @@ btr_cur_pessimistic_insert( bool inherit = false; uint32_t n_reserved = 0; + ut_d(++btr_cur_pessimistic_insert_calls); + ut_ad(dtuple_check_typed(entry)); ut_ad(thr || !(~flags & (BTR_NO_LOCKING_FLAG | BTR_NO_UNDO_LOG_FLAG))); @@ -3602,10 +3628,16 @@ btr_cur_optimistic_update( goto func_exit; } - if (UNIV_UNLIKELY(page_get_data_size(page) + if (UNIV_UNLIKELY((!btr_cur_reduce_pessimistic_update_fallbacks + || new_rec_size < old_rec_size) + && page_get_data_size(page) - old_rec_size + new_rec_size < BTR_CUR_PAGE_COMPRESS_LIMIT(index))) { - /* The page would become too empty */ + /* The page would become too empty. When + innodb_reduce_pessimistic_update_fallbacks is enabled, only + treat this as DB_UNDERFLOW if the record is actually + shrinking; otherwise a freshly split page would be re-merged + even though the update is growing the record. */ err = DB_UNDERFLOW; goto func_exit; } @@ -3797,6 +3829,8 @@ btr_cur_pessimistic_update( block = btr_cur_get_block(cursor); index = cursor->index(); + ut_d(++btr_cur_pessimistic_update_calls); + ut_ad(mtr->memo_contains_flagged(&index->lock, MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr->memo_contains_flagged(block, MTR_MEMO_PAGE_X_FIX)); @@ -3822,6 +3856,9 @@ btr_cur_pessimistic_update( cursor, offsets, offsets_heap, update, cmpl_info, thr, trx_id, mtr); + ut_d(btr_cur_pessimistic_update_optim_err_underflows += (err == DB_UNDERFLOW)); + ut_d(btr_cur_pessimistic_update_optim_err_overflows += (err == DB_OVERFLOW)); + switch (err) { case DB_ZIP_OVERFLOW: case DB_UNDERFLOW: @@ -3992,8 +4029,33 @@ btr_cur_pessimistic_update( goto return_after_reservations; } - rec = btr_cur_insert_if_possible(cursor, new_entry, - offsets, offsets_heap, n_ext, mtr); + /* Force a page split instead of the in-place reinsert below when: + - sysvar opt-in; + - optimistic update returned DB_OVERFLOW (the fit problem this + optimization addresses; DB_UNDERFLOW means the page is too sparse + and the legacy compress path is the right answer); + - uncompressed page: the reorganize-fit check uses uncompressed-page + accounting and is not meaningful for ROW_FORMAT=COMPRESSED; + - page still has at least one record after the delete above (so + the forced split would produce at least 1+1, not 0+1); + - the new record is strictly larger than the old one (only + size-growing UPDATEs should pay the cost of an early split); + - the page is too full to satisfy BTR_CUR_PAGE_REORGANIZE_LIMIT + after a reorganize, so the legacy in-place retry would just churn. + rec=NULL falls through to the split path. */ + if (btr_cur_reduce_pessimistic_update_fallbacks + && optim_err == DB_OVERFLOW + && !buf_block_get_page_zip(block) + && page_get_n_recs(block->page.frame) > 0 + && rec_get_converted_size(index, new_entry, n_ext) + > rec_offs_size(*offsets) + && page_get_max_insert_size_after_reorganize( + block->page.frame, 1) < BTR_CUR_PAGE_REORGANIZE_LIMIT) { + rec = NULL; + } else { + rec = btr_cur_insert_if_possible(cursor, new_entry, + offsets, offsets_heap, n_ext, mtr); + } if (rec) { page_cursor->rec = rec; @@ -4509,6 +4571,8 @@ btr_cur_pessimistic_delete( page = buf_block_get_frame(block); index = btr_cur_get_index(cursor); + ut_d(++btr_cur_pessimistic_delete_calls); + ut_ad(flags == 0 || flags == BTR_CREATE_FLAG); ut_ad(!dict_index_is_online_ddl(index) || dict_index_is_clust(index) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4a4792983cb2a..b1b3831016f4a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -890,6 +890,22 @@ static MYSQL_THDVAR_STR(tmpdir, static size_t truncated_status_writes; +#ifdef UNIV_DEBUG +/** Expose an Atomic_counter as a SHOW_ULONGLONG status variable. +The server-internal enum_mysql_show_type only has SHOW_ATOMIC_COUNTER_UINT32_T, +no 64-bit equivalent, so we materialize the value into the SHOW_VAR buffer +ourselves. */ +template *Counter> +static int show_atomic_counter_u64(MYSQL_THD, SHOW_VAR *var, void *buff, + system_status_var *, enum enum_var_type) +{ + var->type= SHOW_ULONGLONG; + var->value= buff; + *static_cast(buff)= *Counter; + return 0; +} +#endif /* UNIV_DEBUG */ + static SHOW_VAR innodb_status_variables[]= { #ifdef BTR_CUR_HASH_ADAPT {"adaptive_hash_hash_searches", &export_vars.innodb_ahi_hit, SHOW_SIZE_T}, @@ -1071,6 +1087,37 @@ static SHOW_VAR innodb_status_variables[]= { /* InnoDB bulk operations */ {"bulk_operations", &export_vars.innodb_bulk_operations, SHOW_SIZE_T}, +#ifdef UNIV_DEBUG + {"btr_cur_n_index_lock_upgrades", + (void*) &show_atomic_counter_u64< + &btr_cur_n_index_lock_upgrades>, + SHOW_SIMPLE_FUNC}, + {"btr_cur_pessimistic_insert_calls", + (void*) &show_atomic_counter_u64< + &btr_cur_pessimistic_insert_calls>, + SHOW_SIMPLE_FUNC}, + {"btr_cur_pessimistic_update_calls", + (void*) &show_atomic_counter_u64< + &btr_cur_pessimistic_update_calls>, + SHOW_SIMPLE_FUNC}, + {"btr_cur_pessimistic_delete_calls", + (void*) &show_atomic_counter_u64< + &btr_cur_pessimistic_delete_calls>, + SHOW_SIMPLE_FUNC}, + {"btr_cur_pessimistic_update_optim_err_underflows", + (void*) &show_atomic_counter_u64< + &btr_cur_pessimistic_update_optim_err_underflows>, + SHOW_SIMPLE_FUNC}, + {"btr_cur_pessimistic_update_optim_err_overflows", + (void*) &show_atomic_counter_u64< + &btr_cur_pessimistic_update_optim_err_overflows>, + SHOW_SIMPLE_FUNC}, + {"mtr_n_index_x_lock_calls", + (void*) &show_atomic_counter_u64< + &mtr_n_index_x_lock_calls>, + SHOW_SIMPLE_FUNC}, +#endif /* UNIV_DEBUG */ + {NullS, NullS, SHOW_LONG} }; @@ -19811,6 +19858,20 @@ static MYSQL_SYSVAR_ENUM(default_row_format, innodb_default_row_format, NULL, NULL, DEFAULT_ROW_FORMAT_DYNAMIC, &innodb_default_row_format_typelib); +static MYSQL_SYSVAR_BOOL(reduce_pessimistic_update_fallbacks, + btr_cur_reduce_pessimistic_update_fallbacks, PLUGIN_VAR_OPCMDARG, + "Enable the btr_cur_*_update() optimizations when the" + " UPDATE is increasing the size of a record: gate the" + " DB_UNDERFLOW return of btr_cur_optimistic_update() behind actual" + " record shrinkage so freshly split pages are not re-merged when an" + " update is growing a record, and skip the in-place reinsert in" + " btr_cur_pessimistic_update() when an uncompressed page cannot" + " satisfy BTR_CUR_PAGE_REORGANIZE_LIMIT after reorganize, falling" + " through to a page split instead. Trades some B-tree self-healing" + " for a lower rate of pessimistic update fallbacks and the index-lock" + " upgrades they induce", + NULL, NULL, FALSE); + #ifdef UNIV_DEBUG static MYSQL_SYSVAR_UINT(limit_optimistic_insert_debug, btr_cur_limit_optimistic_insert_debug, PLUGIN_VAR_RQCMDARG, @@ -20094,6 +20155,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(compression_failure_threshold_pct), MYSQL_SYSVAR(compression_pad_pct_max), MYSQL_SYSVAR(default_row_format), + MYSQL_SYSVAR(reduce_pessimistic_update_fallbacks), #ifdef UNIV_DEBUG MYSQL_SYSVAR(limit_optimistic_insert_debug), MYSQL_SYSVAR(trx_purge_view_update_only_debug), diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..e669ab60f6203 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -833,8 +833,23 @@ extern ulint btr_cur_n_sea_old; #ifdef UNIV_DEBUG /* Flag to limit optimistic insert records */ extern uint btr_cur_limit_optimistic_insert_debug; +/** Number of times index lock was upgraded from SX to X */ +extern Atomic_counter btr_cur_n_index_lock_upgrades; +/** Number of times btr_cur_pessimistic_insert() was called */ +extern Atomic_counter btr_cur_pessimistic_insert_calls; +/** Number of times btr_cur_pessimistic_update() was called */ +extern Atomic_counter btr_cur_pessimistic_update_calls; +/** Number of times btr_cur_pessimistic_delete() was called */ +extern Atomic_counter btr_cur_pessimistic_delete_calls; +/** Number of times DB_UNDERFLOW was returned as optimistic update error in btr_cur_pessimistic_update() */ +extern Atomic_counter btr_cur_pessimistic_update_optim_err_underflows; +/** Number of times DB_OVERFLOW was returned as optimistic update error in btr_cur_pessimistic_update() */ +extern Atomic_counter btr_cur_pessimistic_update_optim_err_overflows; #endif /* UNIV_DEBUG */ +/** innodb_reduce_pessimistic_update_fallbacks; see the comment in btr0cur.cc */ +extern my_bool btr_cur_reduce_pessimistic_update_fallbacks; + #include "btr0cur.inl" #endif diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index b583919dee0f4..d8add3e4bb2a1 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -41,13 +41,18 @@ Created 11/26/1995 Heikki Tuuri @return old mode */ #define mtr_set_log_mode(m, d) (m)->set_log_mode((d)) +#ifdef UNIV_DEBUG +/** Number of times mtr_x_lock_index() was called. */ +extern Atomic_counter mtr_n_index_x_lock_calls; +#endif + #ifdef UNIV_PFS_RWLOCK # define mtr_s_lock_index(i,m) (m)->s_lock(__FILE__, __LINE__, &(i)->lock) -# define mtr_x_lock_index(i,m) (m)->x_lock(__FILE__, __LINE__, &(i)->lock) +# define mtr_x_lock_index(i,m) do { (m)->x_lock(__FILE__, __LINE__, &(i)->lock); ut_d(++mtr_n_index_x_lock_calls); } while (0) # define mtr_sx_lock_index(i,m) (m)->u_lock(__FILE__, __LINE__, &(i)->lock) #else # define mtr_s_lock_index(i,m) (m)->s_lock(&(i)->lock) -# define mtr_x_lock_index(i,m) (m)->x_lock(&(i)->lock) +# define mtr_x_lock_index(i,m) do { (m)->x_lock(&(i)->lock); ut_d(++mtr_n_index_x_lock_calls); } while (0) # define mtr_sx_lock_index(i,m) (m)->u_lock(&(i)->lock) #endif diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 2cdf2d835795a..cbd3f6cdd535b 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -44,6 +44,11 @@ void (*mtr_t::commit_logger)(mtr_t *, std::pair); std::pair (*mtr_t::finisher)(mtr_t *, size_t); +#ifdef UNIV_DEBUG +/** Number of times mtr_x_lock_index() was called */ +Atomic_counter mtr_n_index_x_lock_calls{0}; +#endif + void mtr_t::finisher_update() { ut_ad(log_sys.latch_have_wr());