MDEV-37723 In MDEV-36861, analyze Q4

This is split from MDEV-36861 where we are investigating performance
issues comparing 10.11 with 11.4

Here we are adding in a missing component to push up the cost of a disk
based temp table lookup.  This is needed to change the plan for Query 4
of the TPROCH, HammerDB performance suite test.

Included test is query 4 translated into our test format and
populated such that it exceeds a lowered in-memory temporary table
threshold.

Patch provided by Monty, test provided by Rex

Author: mariadb-RexJohnston

mysql-test/main/costs.result

@@ -124,3 +124,76 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	range	l_suppkey	l_suppkey	10	NULL	1	Using where; Using index
 drop table t1;
 set global InnoDB.optimizer_disk_read_ratio=@save;
+#
+# MDEV-37723 In MDEV-36861, analyze Q4
+# Alter disk based tmp table lookup cost formula
+#
+create table t1 (a int primary key,
+b date, c char(15),
+d blob,
+key t1_ix1 (b)) ENGINE= InnoDB;
+create table t2 (e int not null,
+f int not null,
+g date, h date,
+primary key (e, f)) ENGINE= InnoDB;
+insert into t1 select seq, date('1993-06-01') + interval seq second,
+chr(65+mod(seq, 4)), NULL from seq_1_to_7000;
+insert into t2 select a.seq, b.seq, date('1993-06-01') + interval b.seq day,
+if (mod(a.seq,2), date('1993-06-01') + interval b.seq+1 day,
+date('1993-06-01') - interval b.seq-1 day)
+from seq_1_to_7000 a, seq_1_to_3 b;
+set @@max_heap_table_size=16384;
+explain format=json select c, count(*) as dc
+from t1
+where b >= date '1993-06-01'
+    and b < date '1993-06-01' + interval '3' month
+and exists
+(
+select * from t2 where e = a and g < h
+)
+group by c
+order by c;
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "cost": 12.53886553,
+    "filesort": {
+      "sort_key": "t1.c",
+      "temporary_table": {
+        "nested_loop": [
+          {
+            "table": {
+              "table_name": "t1",
+              "access_type": "ALL",
+              "possible_keys": ["PRIMARY", "t1_ix1"],
+              "loops": 1,
+              "rows": 7000,
+              "cost": 1.1518548,
+              "filtered": 100,
+              "attached_condition": "t1.b >= DATE'1993-06-01' and t1.b < <cache>(DATE'1993-06-01' + interval '3' month)"
+            }
+          },
+          {
+            "table": {
+              "table_name": "t2",
+              "access_type": "ref",
+              "possible_keys": ["PRIMARY"],
+              "key": "PRIMARY",
+              "key_length": "4",
+              "used_key_parts": ["e"],
+              "ref": ["test.t1.a"],
+              "loops": 7000,
+              "rows": 1,
+              "cost": 6.7528092,
+              "filtered": 100,
+              "attached_condition": "t2.g < t2.h",
+              "first_match": "t1"
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+drop table t1, t2;

mysql-test/main/costs.test

@@ -114,3 +114,38 @@ explain select count(*) from test.t1 force index (l_suppkey) where l_suppkey >=
 drop table t1;
 
 set global InnoDB.optimizer_disk_read_ratio=@save;
+
+--echo #
+--echo # MDEV-37723 In MDEV-36861, analyze Q4
+--echo # Alter disk based tmp table lookup cost formula
+--echo #
+
+create table t1 (a int primary key,
+                 b date, c char(15),
+                 d blob,
+                 key t1_ix1 (b)) ENGINE= InnoDB;
+create table t2 (e int not null,
+                 f int not null,
+                 g date, h date,
+                 primary key (e, f)) ENGINE= InnoDB;
+--disable_warnings
+insert into t1 select seq, date('1993-06-01') + interval seq second,
+  chr(65+mod(seq, 4)), NULL from seq_1_to_7000;
+insert into t2 select a.seq, b.seq, date('1993-06-01') + interval b.seq day,
+  if (mod(a.seq,2), date('1993-06-01') + interval b.seq+1 day,
+    date('1993-06-01') - interval b.seq-1 day)
+  from seq_1_to_7000 a, seq_1_to_3 b;
+
+set @@max_heap_table_size=16384;
+explain format=json select c, count(*) as dc
+  from t1
+  where b >= date '1993-06-01'
+    and b < date '1993-06-01' + interval '3' month
+    and exists
+    (
+      select * from t2 where e = a and g < h
+    )
+  group by c
+  order by c;
+
+drop table t1, t2;

mysql-test/main/opt_trace.result

@@ -12172,8 +12172,8 @@ JS
             "org_cost": 1.159965,
             "postjoin_cost": 4.020888502,
             "one_splitting_cost": 5.180853502,
-            "unsplit_postjoin_cost": 32.78652054,
-            "unsplit_cost": 148.7830205,
+            "unsplit_postjoin_cost": 159.1588125,
+            "unsplit_cost": 275.1553125,
             "rows": 100,
             "refills": 5,
             "total_splitting_cost": 25.90426751,

mysql-test/main/subselect3.result

@@ -1167,9 +1167,9 @@ set @@optimizer_switch='firstmatch=off,materialization=off';
 set @@max_heap_table_size= 16384;
 explain select count(*) from t0 A, t0 B, t0 C, t0 D where D.a in (select a from t1 E where a+1 < 10000 + A.a + B.a +C.a+D.a);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	E	ALL	NULL	NULL	NULL	NULL	5	Start temporary
-1	PRIMARY	A	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
+1	PRIMARY	A	ALL	NULL	NULL	NULL	NULL	10	
 1	PRIMARY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
+1	PRIMARY	E	ALL	NULL	NULL	NULL	NULL	5	Start temporary; Using join buffer (flat, BNL join)
 1	PRIMARY	C	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
 1	PRIMARY	D	ALL	NULL	NULL	NULL	NULL	10	Using where; End temporary; Using join buffer (flat, BNL join)
 flush status;

sql/opt_subselect.cc

@@ -2779,12 +2779,14 @@ get_tmp_table_costs(THD *thd, double row_count, uint row_size, bool blobs_used,
                            tmp_table_optimizer_costs.row_copy_cost :
                            0);
     /* Disk based table */
-    cost.lookup=          ((tmp_table_optimizer_costs.key_lookup_cost *
+    cost.lookup=          ((tmp_table_optimizer_costs.key_lookup_cost +
+                            tmp_table_optimizer_costs.disk_read_cost *
                             tmp_table_optimizer_costs.disk_read_ratio) +
                            row_copy_cost);
     cost.write=           cost.lookup;
     cost.create=          DISK_TEMPTABLE_CREATE_COST;
     cost.block_size=      DISK_TEMPTABLE_BLOCK_SIZE;
+    /* The following costs are only used for table scans */
     cost.avg_io_cost=     tmp_table_optimizer_costs.disk_read_cost;
     cost.cache_hit_ratio= tmp_table_optimizer_costs.disk_read_ratio;
   }