MDEV-25015 Custom formatting of strings in MariaDB queries

SFORMAT() SQL function that uses fmtlib (https://fmt.dev/)
for python-like (also Rust, C++20, etc) string formatting

Only fmtlib 7.0.0+ is supported, older fmtlib
produces different results in the test.

No native support for temporal and decimal values,
* TIME_RESULT is handled as STRING_RESULT
* DECIMAL_RESULT as REAL_RESULT

.gitignore

@@ -103,6 +103,7 @@ packaging/rpm-oel/mysql.spec
 packaging/rpm-uln/mysql.10.0.11.spec
 packaging/solaris/postinstall-solaris
 extra/pcre2
+extra/libfmt
 plugin/auth_pam/auth_pam_tool
 plugin/auth_pam/config_auth_pam.h
 plugin/aws_key_management/aws-sdk-cpp

CMakeLists.txt

@@ -158,6 +158,7 @@ INCLUDE(readline)
 INCLUDE(libutils)
 INCLUDE(dtrace)
 INCLUDE(pcre)
+INCLUDE(libfmt)
 INCLUDE(ctest)
 INCLUDE(plugin)
 INCLUDE(install_macros)
@@ -382,6 +383,7 @@ MYSQL_CHECK_READLINE()
 SET(MALLOC_LIBRARY "system")
 
 CHECK_PCRE()
+CHECK_LIBFMT()
 ADD_SUBDIRECTORY(tpool)
 CHECK_SYSTEMD()
 

cmake/libfmt.cmake

+INCLUDE (CheckCXXSourceCompiles)
+INCLUDE (ExternalProject)
+
+SET(WITH_LIBFMT "auto" CACHE STRING
+   "Which libfmt to use (possible values are 'bundled', 'system', or 'auto')")
+
+MACRO(BUNDLE_LIBFMT)
+  SET(dir "${CMAKE_BINARY_DIR}/extra/libfmt")
+  SET(LIBFMT_INCLUDE_DIR "${dir}/src/libfmt/include")
+  ADD_LIBRARY(fmt STATIC IMPORTED GLOBAL)
+  SET(file ${dir}/src/libfmt-build/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}fmt${CMAKE_STATIC_LIBRARY_SUFFIX})
+  SET_TARGET_PROPERTIES(fmt PROPERTIES IMPORTED_LOCATION ${file})
+
+  ExternalProject_Add(
+    libfmt
+    PREFIX   "${dir}"
+    URL      "https://github.com/fmtlib/fmt/archive/refs/tags/8.0.1.zip"
+    URL_MD5  e77873199e897ca9f780479ad68e25b1
+    INSTALL_COMMAND ""
+    CMAKE_ARGS
+      "-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
+      "-DBUILD_SHARED_LIBS=OFF"
+      "-DFMT_DEBUG_POSTFIX="
+      "-DFMT_DOC=OFF"
+      "-DFMT_TEST=OFF"
+      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+      "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} ${PIC_FLAG}"
+      "-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}"
+      "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=${CMAKE_CXX_FLAGS_RELWITHDEBINFO}"
+      "-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}"
+      "-DCMAKE_CXX_FLAGS_MINSIZEREL=${CMAKE_CXX_FLAGS_MINSIZEREL}"
+      "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+    BUILD_BYPRODUCTS ${file}
+  )
+  SET_TARGET_PROPERTIES(fmt PROPERTIES EXCLUDE_FROM_ALL TRUE)
+ENDMACRO()
+
+MACRO (CHECK_LIBFMT)
+  IF(WITH_LIBFMT STREQUAL "system" OR WITH_LIBFMT STREQUAL "auto")
+    SET(CMAKE_REQUIRED_LIBRARIES fmt)
+    CHECK_CXX_SOURCE_COMPILES(
+    "#include <fmt/core.h>
+     #include <iostream>
+     int main() {
+       std::cout << fmt::format(\"The answer is {}.\", 42);
+     }" HAVE_SYSTEM_LIBFMT)
+    SET(CMAKE_REQUIRED_LIBRARIES)
+  ENDIF()
+  IF(NOT HAVE_SYSTEM_LIBFMT OR WITH_LIBFMT STREQUAL "bundled")
+    IF (WITH_LIBFMT STREQUAL "system")
+      MESSAGE(FATAL_ERROR "system libfmt library is not found")
+    ENDIF()
+    BUNDLE_LIBFMT()
+  ELSE()
+    FIND_FILE(Libfmt_core_h fmt/core.h) # for build_depends.cmake
+  ENDIF()
+ENDMACRO()
+
+MARK_AS_ADVANCED(LIBFMT_INCLUDE_DIR)

libmysqld/CMakeLists.txt

@@ -24,6 +24,7 @@ ${CMAKE_SOURCE_DIR}/sql
 ${CMAKE_SOURCE_DIR}/tpool
 ${CMAKE_BINARY_DIR}/sql 
 ${PCRE_INCLUDES}
+${LIBFMT_INCLUDE_DIR}
 ${ZLIB_INCLUDE_DIR}
 ${SSL_INCLUDE_DIRS}
 ${SSL_INTERNAL_INCLUDE_DIRS}
@@ -395,9 +396,10 @@ IF(NOT DISABLE_SHARED)
     # libmysqld
     SET_TARGET_PROPERTIES(libmysqld PROPERTIES CLEAN_DIRECT_OUTPUT 1)
     SET_TARGET_PROPERTIES(mysqlserver PROPERTIES CLEAN_DIRECT_OUTPUT 1)
-    TARGET_LINK_LIBRARIES(mysqlserver LINK_PRIVATE tpool ${CRC32_LIBRARY})
+    TARGET_LINK_LIBRARIES(mysqlserver LINK_PRIVATE tpool fmt ${CRC32_LIBRARY})
     IF(LIBMYSQLD_SO_EXTRA_LIBS)
        TARGET_LINK_LIBRARIES(libmysqld LINK_PRIVATE ${LIBMYSQLD_SO_EXTRA_LIBS})
     ENDIF()
   ENDIF()
 ENDIF()
+

sql/CMakeLists.txt

@@ -52,6 +52,7 @@ ENDIF()
 INCLUDE_DIRECTORIES(
 ${CMAKE_SOURCE_DIR}/include
 ${CMAKE_SOURCE_DIR}/sql
+${LIBFMT_INCLUDE_DIR}
 ${PCRE_INCLUDES}
 ${ZLIB_INCLUDE_DIR}
 ${SSL_INCLUDE_DIRS}
@@ -207,7 +208,7 @@ MAYBE_DISABLE_IPO(sql)
 DTRACE_INSTRUMENT(sql)
 TARGET_LINK_LIBRARIES(sql
   mysys mysys_ssl dbug strings vio pcre2-8
-  tpool
+  tpool fmt
   ${LIBWRAP} ${LIBCRYPT} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT}
   ${SSL_LIBRARIES}
   ${LIBSYSTEMD})

sql/item_create.cc

@@ -1922,6 +1922,16 @@ class Create_func_sec_to_time : public Create_func_arg1
   virtual ~Create_func_sec_to_time() {}
 };
 
+class Create_func_sformat : public Create_native_func
+{
+public:
+  virtual Item *create_native(THD *thd, LEX_CSTRING *name,
+			      List<Item> *item_list);
+  static Create_func_sformat s_singleton;
+protected:
+  Create_func_sformat() {}
+  virtual ~Create_func_sformat() {}
+};
 
 class Create_func_sha : public Create_func_arg1
 {
@@ -5010,6 +5020,26 @@ Create_func_sec_to_time::create_1_arg(THD *thd, Item *arg1)
   return new (thd->mem_root) Item_func_sec_to_time(thd, arg1);
 }
 
+Create_func_sformat Create_func_sformat::s_singleton;
+
+Item*
+Create_func_sformat::create_native(THD *thd, LEX_CSTRING *name,
+                                  List<Item> *item_list)
+{
+  int arg_count= 0;
+  
+  if (item_list != NULL)
+    arg_count= item_list->elements;
+  
+  if (unlikely(arg_count < 1))
+  {
+    my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str);
+    return NULL;
+  }
+  
+  return new (thd->mem_root) Item_func_sformat(thd, *item_list);
+}
+
 
 Create_func_sha Create_func_sha::s_singleton;
 
@@ -5677,6 +5707,7 @@ Native_func_registry func_array[] =
   { { STRING_WITH_LEN("RTRIM") }, BUILDER(Create_func_rtrim)},
   { { STRING_WITH_LEN("RTRIM_ORACLE") }, BUILDER(Create_func_rtrim_oracle)},
   { { STRING_WITH_LEN("SEC_TO_TIME") }, BUILDER(Create_func_sec_to_time)},
+  { { STRING_WITH_LEN("SFORMAT") }, BUILDER(Create_func_sformat)},
   { { STRING_WITH_LEN("SHA") }, BUILDER(Create_func_sha)},
   { { STRING_WITH_LEN("SHA1") }, BUILDER(Create_func_sha)},
   { { STRING_WITH_LEN("SHA2") }, BUILDER(Create_func_sha2)},

sql/item_strfunc.cc

@@ -55,6 +55,9 @@ C_MODE_END
 #include <sql_repl.h>
 #include "sql_statistics.h"
 
+/* fmtlib include (https://fmt.dev/). */
+#include "fmt/format.h"
+
 size_t username_char_length= USERNAME_CHAR_LENGTH;
 
 /*
@@ -1302,6 +1305,106 @@ bool Item_func_replace::fix_length_and_dec()
 }
 
 
+bool Item_func_sformat::fix_length_and_dec()
+{
+  ulonglong char_length= 0;
+  
+  if (agg_arg_charsets_for_string_result(collation, args, arg_count))
+    return TRUE;
+  
+  for (uint i=0 ; i < arg_count ; i++)
+    char_length+= args[i]->max_char_length();
+  
+  fix_char_length_ulonglong(char_length);
+  return FALSE;
+}
+
+/*
+ * SFORMAT(format_string, ...)
+ * This function receives a formatting specification string and N parameters
+ * (N >= 0), and it returns string formatted using the rules the user passed 
+ * in the specification. It uses fmtlib (https://fmt.dev/).
+ */
+String *Item_func_sformat::val_str(String *res)
+{
+  DBUG_ASSERT(fixed());
+  uint                          carg=    1;
+  using                         ctx=     fmt::format_context;
+  String                       *fmt_arg= NULL;
+  String                       *parg=    NULL;
+  String                       *val_arg= NULL;
+  fmt::format_args::format_arg *vargs=   NULL;
+  
+  null_value= true;
+  if (!(fmt_arg= args[0]->val_str(res)))
+    return NULL;
+  
+  if (!(vargs= new fmt::format_args::format_arg[arg_count - 1]))
+    return NULL;
+  
+  if (!(val_arg= new String[arg_count - 1]))
+  {
+    delete [] vargs;
+    return NULL;
+  }
+  
+  /* Creates the array of arguments for vformat */
+  while (carg < arg_count)
+  {
+    switch (args[carg]->result_type())
+    {
+    case INT_RESULT:
+      vargs[carg-1]= fmt::detail::make_arg<ctx>(args[carg]->val_int());
+      break;
+    case DECIMAL_RESULT:
+    case REAL_RESULT:
+      vargs[carg-1]= fmt::detail::make_arg<ctx>(args[carg]->val_real());
+      break;
+    case STRING_RESULT:
+      if (!(parg= args[carg]->val_str(&val_arg[carg-1])))
+      {
+        delete [] vargs;
+        delete [] val_arg;
+        return NULL;
+      }
+      if (parg->length() == 1)
+        vargs[carg-1]= fmt::detail::make_arg<ctx>(parg->ptr()[0]);
+      else
+        vargs[carg-1]= fmt::detail::make_arg<ctx>(parg->c_ptr_safe());
+      break;
+    default:
+      DBUG_ASSERT(0);
+      delete [] vargs;
+      delete [] val_arg;
+      return NULL;
+    }
+    carg++;
+  }
+  
+  null_value= false;
+  /* Create the string output  */
+  try
+  {
+    auto text = fmt::vformat(fmt_arg->c_ptr_safe(),
+                             fmt::format_args(vargs, arg_count-1));
+    res->set("", 0, collation.collation);
+    res->append(text.c_str(), text.size());
+  }
+  catch (const fmt::format_error &ex)
+  {
+    THD *thd= current_thd;
+    push_warning_printf(thd,
+                        Sql_condition::WARN_LEVEL_WARN,
+                        WARN_WRONG_FMTLIB_FORMAT,
+                        ER_THD(thd, WARN_WRONG_FMTLIB_FORMAT),
+                        ex.what());
+    null_value= true;
+  }
+  delete [] vargs;
+  delete [] val_arg;
+  return null_value ? NULL : res;
+}
+
 /*********************************************************************/
 bool Item_func_regexp_replace::fix_length_and_dec()
 {

sql/item_strfunc.h

@@ -586,6 +586,22 @@ class Item_func_substr :public Item_str_func
   { return get_item_copy<Item_func_substr>(thd, this); }
 };
 
+class Item_func_sformat :public Item_str_func
+{
+public:
+  Item_func_sformat(THD *thd, List<Item> &list):
+    Item_str_func(thd, list) { }
+  String *val_str(String*) override;
+  bool fix_length_and_dec() override;
+  LEX_CSTRING func_name_cstring() const override
+  {
+    static LEX_CSTRING name= {STRING_WITH_LEN("sformat") };
+    return name;
+  }
+  Item *get_copy(THD *thd) override
+  { return get_item_copy<Item_func_sformat>(thd, this); }
+};
+
 class Item_func_substr_oracle :public Item_func_substr
 {
 protected:

sql/share/errmsg-utf8.txt

@@ -7992,3 +7992,5 @@ ER_REMOVED_ORPHAN_TRIGGER
         eng "Dropped orphan trigger '%-.64s', originally created for table: '%-.192s'"
 ER_STORAGE_ENGINE_DISABLED
         eng "Storage engine %s is disabled"
+WARN_WRONG_FMTLIB_FORMAT
+        eng "Wrong format exception: %-.64s"