MDEV-38199 Optimizer Error with = SOME on UNIQUE Column Using Decimal/Integer Types

return 2 (= value was truncated) when a decimal or float number
was modified when stored in Field_long (= fractional part was lost).

This tells the optimizer that a number with a non-zero fractional
part cannot be found in an index over an integer field.

Author: vuvova

mysql-test/main/null.result

@@ -1647,18 +1647,18 @@ DROP TABLE t1;
 #
 CREATE TABLE t1 (a YEAR);
 INSERT INTO t1 VALUES (2010),(2020);
-SELECT * FROM t1 WHERE a=2010 AND NULLIF(10.1,a) IS NULL;
+SELECT * FROM t1 WHERE a=2010 AND NULLIF(10,a) IS NULL;
 a
 2010
-EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND NULLIF(10.1,a) IS NULL;
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND NULLIF(10,a) IS NULL;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 2010
-SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10.1=a THEN NULL ELSE 10.1 END IS NULL;
+SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10=a THEN NULL ELSE 10 END IS NULL;
 a
 2010
-EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10.1=a THEN NULL ELSE 10.1 END IS NULL;
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10=a THEN NULL ELSE 10 END IS NULL;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
@@ -1708,20 +1708,21 @@ a
 2010
 SELECT * FROM t1 WHERE NULLIF(a,2011.1)='2011';
 a
+2011
 SELECT * FROM t1 WHERE a=10 AND NULLIF(a,2011.1)='2011';
 a
 EXPLAIN EXTENDED
 SELECT * FROM t1 WHERE a=10 AND NULLIF(a,2011.1)='2011';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
-Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 2010 and (case when 2010 = 2011 then NULL else `test`.`t1`.`a` end) = '2011'
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 2010 and nullif(`test`.`t1`.`a`,2011.1) = '2011'
 EXPLAIN EXTENDED
 SELECT * FROM t1 WHERE a=10 AND NULLIF(a,2011.1)=CONCAT('2011',RAND());
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 Warnings:
-Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 2010 and (case when 2010 = 2011 then NULL else `test`.`t1`.`a` end) = concat('2011',rand())
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = 2010 and nullif(`test`.`t1`.`a`,2011.1) = concat('2011',rand())
 DROP TABLE t1;
 #
 # MDEV-8754 Wrong result for SELECT..WHERE year_field=2020 AND NULLIF(year_field,2010)='2020'

mysql-test/main/null.test

@@ -998,10 +998,10 @@ DROP TABLE t1;
 --echo #
 CREATE TABLE t1 (a YEAR);
 INSERT INTO t1 VALUES (2010),(2020);
-SELECT * FROM t1 WHERE a=2010 AND NULLIF(10.1,a) IS NULL;
-EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND NULLIF(10.1,a) IS NULL;
-SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10.1=a THEN NULL ELSE 10.1 END IS NULL;
-EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10.1=a THEN NULL ELSE 10.1 END IS NULL;
+SELECT * FROM t1 WHERE a=2010 AND NULLIF(10,a) IS NULL;
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND NULLIF(10,a) IS NULL;
+SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10=a THEN NULL ELSE 10 END IS NULL;
+EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010 AND CASE WHEN 10=a THEN NULL ELSE 10 END IS NULL;
 DROP TABLE t1;
 
 --echo # Two warnings expected

mysql-test/main/table_elim.result

@@ -345,7 +345,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 explain select t1.a from t1 left join t2 on t2.pk between 0.5 and 1.5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	
-1	SIMPLE	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using where; Using index
+1	SIMPLE	t2	index	PRIMARY	PRIMARY	4	NULL	2	Using where; Using index
 explain select t1.a from t1 left join t2 on t2.pk between 10 and 10;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	

mysql-test/main/type_decimal.result

@@ -349,7 +349,6 @@ Warning	1264	Out of range value for column 'a' at row 3
 insert ignore into t1 values (1e+100),(1e-100),(-1e+100);
 Warnings:
 Warning	1264	Out of range value for column 'a' at row 1
-Note	1265	Data truncated for column 'a' at row 2
 Warning	1264	Out of range value for column 'a' at row 3
 insert into t1 values (123.4e0),(123.4e+2),(123.4e-2),(123e1),(123e+0);
 Warnings:
@@ -1700,6 +1699,19 @@ a
 123.47
 DROP TABLE t1;
 DROP PROCEDURE show_table;
-#
 # End of 10.11 tests
 #
+# MDEV-38199 Optimizer Error with = SOME on UNIQUE Column Using Decimal/Integer Types
+#
+create table t1 (i int, key (i));
+insert into t1 values(1),(1),(1);
+select i from t1 where i=0.6;
+i
+select i from t1 ignore index(i) where i=0.6;
+i
+select i from t1 where i=0.6e0;
+i
+select i from t1 ignore index(i) where i=0.6e0;
+i
+drop table t1;
+# End of 12.3 tests

mysql-test/main/type_decimal.test

@@ -930,7 +930,17 @@ DROP TABLE t1;
 
 DROP PROCEDURE show_table;
 
+--echo # End of 10.11 tests
 
 --echo #
---echo # End of 10.11 tests
+--echo # MDEV-38199 Optimizer Error with = SOME on UNIQUE Column Using Decimal/Integer Types
 --echo #
+create table t1 (i int, key (i));
+insert into t1 values(1),(1),(1);
+select i from t1 where i=0.6;
+select i from t1 ignore index(i) where i=0.6;
+select i from t1 where i=0.6e0;
+select i from t1 ignore index(i) where i=0.6e0;
+drop table t1;
+
+--echo # End of 12.3 tests

mysql-test/main/type_year.result

@@ -229,16 +229,12 @@ yyyy	c4
 1999	1999
 SELECT * FROM t2 WHERE yy = 1999.1;
 yy	c2
-99	1999
 SELECT * FROM t4 WHERE yyyy = 1999.1;
 yyyy	c4
-1999	1999
 SELECT * FROM t2 WHERE yy = 1998.9;
 yy	c2
-99	1999
 SELECT * FROM t4 WHERE yyyy = 1998.9;
 yyyy	c4
-1999	1999
 # Coverage tests for YEAR with zero/2000 constants:
 SELECT * FROM t2 WHERE yy = 0;
 yy	c2

mysql-test/suite/innodb/r/innodb-isolation.result

@@ -973,7 +973,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 2	SUBQUERY	t1	index	NULL	k2	5	NULL	#	Using index
 EXPLAIN SELECT COUNT(*) FROM t1 WHERE c1 > (SELECT AVG(c1) FROM t1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t1	range	PRIMARY	PRIMARY	4	NULL	#	Using where
+1	PRIMARY	t1	index	PRIMARY	k2	5	NULL	#	Using where; Using index
 2	SUBQUERY	t1	index	NULL	k2	5	NULL	#	Using index
 #
 # Make all indexes in t2 obsolete to the active repeatable read transaction

sql/field.cc

@@ -1897,11 +1897,6 @@ int Field::warn_if_overflow(int op_result)
     set_warning(ER_WARN_DATA_OUT_OF_RANGE, 1);
     return 1;
   }
-  if (op_result == E_DEC_TRUNCATED)
-  {
-    set_note(WARN_DATA_TRUNCATED, 1);
-    /* We return 0 here as this is not a critical issue */
-  }
   return 0;
 }
 
@@ -2254,31 +2249,28 @@ longlong Field::convert_decimal2longlong(const my_decimal *val,
                                          bool unsigned_flag, int *err)
 {
   longlong i;
+  int res= E_DEC_OVERFLOW;
   if (unsigned_flag)
   {
     if (val->sign())
     {
       set_warning(ER_WARN_DATA_OUT_OF_RANGE, 1);
       i= 0;
-      *err= 1;
     }
-    else if (warn_if_overflow(my_decimal2int((E_DEC_ERROR &
-                                              ~E_DEC_OVERFLOW &
-                                              ~E_DEC_TRUNCATED),
-                                             val, TRUE, &i)))
+    else if (warn_if_overflow(res= my_decimal2int(E_DEC_FATAL_ERROR &
+                                                  ~E_DEC_OVERFLOW,
+                                                  val, TRUE, &i)))
     {
       i= ~(longlong) 0;
-      *err= 1;
     }
   }
-  else if (warn_if_overflow(my_decimal2int((E_DEC_ERROR &
-                                            ~E_DEC_OVERFLOW &
-                                            ~E_DEC_TRUNCATED),
-                                           val, FALSE, &i)))
+  else if (warn_if_overflow(res= my_decimal2int(E_DEC_FATAL_ERROR &
+                                                ~E_DEC_OVERFLOW,
+                                                val, FALSE, &i)))
   {
     i= (val->sign() ? LONGLONG_MIN : LONGLONG_MAX);
-    *err= 1;
   }
+  *err= res == E_DEC_TRUNCATED ? 2 : res != 0;
   return i;
 }
 
@@ -4519,12 +4511,12 @@ int Field_long::store(const char *from,size_t len,CHARSET_INFO *cs)
 }
 
 
-int Field_long::store(double nr)
+int Field_long::store(double val)
 {
   DBUG_ASSERT(marked_for_write_or_computed());
   int error= 0;
   int32 res;
-  nr=rint(nr);
+  double nr= rint(val);
   if (unsigned_flag)
   {
     if (nr < 0)
@@ -4558,6 +4550,8 @@ int Field_long::store(double nr)
   }
   if (unlikely(error))
     set_warning(ER_WARN_DATA_OUT_OF_RANGE, 1);
+  else if (nr != val)
+    error= 2;
 
   int4store(ptr,res);
   return error;