MDEV-35920: Fix assertion failure "decimals == 0" in Item_func_mod::fix_length_and_dec_int

mysql-test/main/bug35920.result

@@ -0,0 +1,15 @@
+#
+# MDEV-35920: Hex hybrid type causing invalid decimals in arithmetic context
+#
+SELECT MOD(COALESCE(0x30), 1);
+MOD(COALESCE(0x30), 1)
+0
+SELECT MOD(LEAST(0x30,0x30), 1);
+MOD(LEAST(0x30,0x30), 1)
+0
+SELECT MOD(IFNULL(0x30,0x30), 1);
+MOD(IFNULL(0x30,0x30), 1)
+0
+SELECT MOD(IF(0,0x30,0x30), 1);
+MOD(IF(0,0x30,0x30), 1)
+0

mysql-test/main/bug35920.test

@@ -0,0 +1,11 @@
+--echo #
+--echo # MDEV-35920: Hex hybrid type causing invalid decimals in arithmetic context
+--echo #
+
+SELECT MOD(COALESCE(0x30), 1);
+
+SELECT MOD(LEAST(0x30,0x30), 1);
+
+SELECT MOD(IFNULL(0x30,0x30), 1);
+
+SELECT MOD(IF(0,0x30,0x30), 1);

sql/sql_type.cc

@@ -1383,6 +1383,35 @@ aggregate_attributes_string(const LEX_CSTRING &func_name,
 }
 
 
+/**
+  Aggregate result type attributes for hexadecimal/hybrid string functions.
+
+  @param field_type  Field type.
+  @param items       Argument array.
+  @param nitems      Number of arguments.
+
+  @retval            False on success, true on error.
+*/
+bool Type_std_attributes::
+aggregate_attributes_hex_hybrid(const LEX_CSTRING &func_name,
+                                Item **items, uint nitems)
+{
+  if (agg_arg_charsets_for_string_result(collation, func_name,
+                                         items, nitems, 1))
+    return true;
+
+  if (collation.collation == &my_charset_bin)
+    max_length= find_max_octet_length(items, nitems);
+  else
+    fix_char_length(find_max_char_length(items, nitems));
+
+  unsigned_flag= true;
+  decimals= 0;
+
+  return false;
+}
+
+
 /*
   Find a handler by its ODBC literal data type.
 
@@ -4786,6 +4815,17 @@ bool Type_handler_string_result::
   return func->aggregate_attributes_string(func_name, items, nitems);
 }
 
+bool Type_handler_hex_hybrid::
+        Item_hybrid_func_fix_attributes(THD *thd,
+                                        const LEX_CSTRING &name,
+                                        Type_handler_hybrid_field_type *,
+                                        Type_all_attributes *attr,
+                                        Item **items,
+                                        uint nitems) const
+{
+  return attr->aggregate_attributes_hex_hybrid(name, items, nitems);
+}
+
 
 
 /*

sql/sql_type.h

@@ -3359,6 +3359,8 @@ class Type_std_attributes: public Type_numeric_attributes
   }
   bool aggregate_attributes_string(const LEX_CSTRING &func_name,
                                    Item **item, uint nitems);
+  bool aggregate_attributes_hex_hybrid(const LEX_CSTRING &func_name,
+                                       Item **item, uint nitems);
   void aggregate_attributes_temporal(uint int_part_length,
                                      Item **item, uint nitems)
   {
@@ -7395,6 +7397,12 @@ class Type_handler_hex_hybrid: public Type_handler_varchar
   const Type_handler *cast_to_int_type_handler() const override;
   bool Item_func_round_fix_length_and_dec(Item_func_round *) const override;
   bool Item_func_int_val_fix_length_and_dec(Item_func_int_val*) const override;
+  bool Item_hybrid_func_fix_attributes(THD *thd,
+                                       const LEX_CSTRING &name,
+                                       Type_handler_hybrid_field_type *,
+                                       Type_all_attributes *attr,
+                                       Item **items,
+                                       uint nitems) const override;
 };