WL#2575 - Fulltext: Parser plugin for FTS

WL#2763 - MySQL plugin interface: step 1
Manual merge from CNET tree.

include/ft_global.h

@@ -53,6 +53,7 @@ extern ulong ft_min_word_len;
 extern ulong ft_max_word_len;
 extern ulong ft_query_expansion_limit;
 extern char  ft_boolean_syntax[15];
+extern struct st_mysql_ftparser ft_default_parser;
 
 int ft_init_stopwords(void);
 void ft_free_stopwords(void);

include/my_global.h

@@ -1380,4 +1380,23 @@ do { doubleget_union _tmp; \
 #define dlerror() ""
 #endif
 
+#ifdef HAVE_DLOPEN
+#if defined(__WIN__)
+#define dlsym(lib, name) GetProcAddress((HMODULE)lib, name)
+#define dlopen(libname, unused) LoadLibraryEx(libname, NULL, 0)
+#define dlclose(lib) FreeLibrary((HMODULE)lib)
+#elif !defined(OS2)
+#include <dlfcn.h>
+#endif
+#endif
+
+/* FreeBSD 2.2.2 does not define RTLD_NOW) */
+#ifndef RTLD_NOW
+#define RTLD_NOW 1
+#endif
+
+#ifndef HAVE_DLERROR
+#define dlerror() ""
+#endif
+
 #endif /* my_global_h */

include/myisam.h

@@ -32,6 +32,7 @@ extern "C" {
 #include "keycache.h"
 #endif
 #include "my_handler.h"
+#include <plugin.h>
 
 	/* defines used by myisam-funktions */
 
@@ -196,6 +197,7 @@ typedef struct st_mi_keydef		/* Key definition with open & info */
   uint32 version;			/* For concurrent read/write */
 
   HA_KEYSEG *seg,*end;
+  struct st_mysql_ftparser *parser;     /* Fulltext [pre]parser */
   int (*bin_search)(struct st_myisam_info *info,struct st_mi_keydef *keyinfo,
 		    uchar *page,uchar *key,
 		    uint key_len,uint comp_flag,uchar * *ret_pos,

libmysqld/Makefile.am

@@ -20,11 +20,13 @@
 MYSQLDATAdir =		$(localstatedir)
 MYSQLSHAREdir =		$(pkgdatadir)
 MYSQLBASEdir=		$(prefix)
+MYSQLLIBdir=            $(libdir)
 
 DEFS =			-DEMBEDDED_LIBRARY -DMYSQL_SERVER \
 			-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
 			-DDATADIR="\"$(MYSQLDATAdir)\"" \
-			-DSHAREDIR="\"$(MYSQLSHAREdir)\""
+			-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
+			-DLIBDIR="\"$(MYSQLLIBdir)\""
 INCLUDES=		@bdb_includes@ \
 			-I$(top_builddir)/include -I$(top_srcdir)/include \
 			-I$(top_srcdir)/sql -I$(top_srcdir)/sql/examples \

mysql-test/r/connect.result

@@ -9,6 +9,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -36,6 +37,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -71,6 +73,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv

mysql-test/r/information_schema.result

@@ -60,6 +60,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -709,7 +710,7 @@ CREATE TABLE t_crashme ( f1 BIGINT);
 CREATE VIEW a1 (t_CRASHME) AS SELECT f1 FROM t_crashme GROUP BY f1;
 CREATE VIEW a2 AS SELECT t_CRASHME FROM a1;
 count(*)
-101
+102
 drop view a2, a1;
 drop table t_crashme;
 select table_schema,table_name, column_name from
@@ -779,7 +780,7 @@ flush privileges;
 SELECT table_schema, count(*) FROM information_schema.TABLES GROUP BY TABLE_SCHEMA;
 table_schema	count(*)
 information_schema	16
-mysql	17
+mysql	18
 create table t1 (i int, j int);
 create trigger trg1 before insert on t1 for each row
 begin

mysql-test/r/mysqlcheck.result

@@ -6,6 +6,7 @@ mysql.help_keyword                                 OK
 mysql.help_relation                                OK
 mysql.help_topic                                   OK
 mysql.host                                         OK
+mysql.plugin                                       OK
 mysql.proc                                         OK
 mysql.procs_priv                                   OK
 mysql.tables_priv                                  OK
@@ -23,6 +24,7 @@ mysql.help_keyword                                 OK
 mysql.help_relation                                OK
 mysql.help_topic                                   OK
 mysql.host                                         OK
+mysql.plugin                                       OK
 mysql.proc                                         OK
 mysql.procs_priv                                   OK
 mysql.tables_priv                                  OK

mysql-test/r/system_mysql_db.result

@@ -9,6 +9,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv

mysql-test/t/system_mysql_db_fix.test

@@ -85,7 +85,7 @@ INSERT INTO user VALUES ('localhost','',    '','N','N','N','N','N','N','N','N','
 
 -- disable_query_log
 
-DROP TABLE db, host, user, func, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
+DROP TABLE db, host, user, func, plugin, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
 
 -- enable_query_log
 

scripts/mysql_create_system_tables.sh

@@ -39,8 +39,8 @@ c_hc=""
 c_hr="" 
 c_hk="" 
 i_ht=""
-c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls=""
-i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls=""
+c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls="" c_pl=""
+i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls="" i_pl=""
 c_p="" c_pp=""
 
 # Check for old tables
@@ -202,6 +202,21 @@ then
   c_f="$c_f   comment='User defined functions';"
 fi
 
+if test ! -f $mdata/plugin.frm
+then
+  if test "$1" = "verbose" ; then
+    echo "Preparing plugin table" 1>&2;
+  fi
+
+  c_pl="$c_pl CREATE TABLE plugin ("
+  c_pl="$c_pl   name char(64) binary DEFAULT '' NOT NULL,"
+  c_pl="$c_pl   dl char(128) DEFAULT '' NOT NULL,"
+  c_pl="$c_pl   PRIMARY KEY (name)"
+  c_pl="$c_pl ) engine=MyISAM"
+  c_pl="$c_pl CHARACTER SET utf8 COLLATE utf8_bin"
+  c_pl="$c_pl   comment='MySQL plugins';"
+fi
+
 if test ! -f $mdata/tables_priv.frm
 then
   if test "$1" = "verbose" ; then
@@ -741,6 +756,9 @@ $i_u
 $c_f
 $i_f
 
+$c_pl
+$i_pl
+
 $c_t
 $c_c
 

scripts/mysql_fix_privilege_tables.sql

@@ -19,6 +19,12 @@ CREATE TABLE IF NOT EXISTS func (
   PRIMARY KEY (name)
 ) CHARACTER SET utf8 COLLATE utf8_bin;
 
+CREATE TABLE IF NOT EXISTS plugin (
+  name char(64) binary DEFAULT '' NOT NULL,
+  dl char(128) DEFAULT '' NOT NULL,
+  PRIMARY KEY (name)
+) CHARACTER SET utf8 COLLATE utf8_bin;
+
 ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
 
 -- Detect whether or not we had the Grant_priv column

sql/Makefile.am

@@ -19,6 +19,7 @@
 MYSQLDATAdir =		$(localstatedir)
 MYSQLSHAREdir =		$(pkgdatadir)
 MYSQLBASEdir=		$(prefix)
+MYSQLLIBdir=            $(pkglibdir)
 INCLUDES =		@ZLIB_INCLUDES@ \
 			@bdb_includes@ @innodb_includes@ @ndbcluster_includes@ \
 			-I$(top_builddir)/include -I$(top_srcdir)/include \
@@ -116,6 +117,7 @@ DEFS =			-DMYSQL_SERVER \
 			-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
 			-DDATADIR="\"$(MYSQLDATAdir)\"" \
 			-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
+			-DLIBDIR="\"$(MYSQLLIBdir)\"" \
 			@DEFS@
 
 BUILT_SOURCES =		sql_yacc.cc sql_yacc.h lex_hash.h

sql/ha_myisam.cc

@@ -280,6 +280,7 @@ err:
 
 int ha_myisam::open(const char *name, int mode, uint test_if_locked)
 {
+  uint i;
   if (!(file=mi_open(name, mode, test_if_locked)))
     return (my_errno ? my_errno : -1);
   
@@ -292,6 +293,14 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked)
     int_table_flags|=HA_REC_NOT_IN_SEQ;
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
     int_table_flags|=HA_HAS_CHECKSUM;
+
+  for (i= 0; i < table->s->keys; i++)
+  {
+    struct st_plugin_int *parser= table->key_info[i].parser;
+    if (table->key_info[i].flags & HA_USES_PARSER)
+      file->s->keyinfo[i].parser=
+        (struct st_mysql_ftparser *)parser->plugin->info;
+  }
   return (0);
 }
 

sql/lex.h

@@ -246,6 +246,7 @@ static SYMBOL symbols[] = {
   { "INSENSITIVE",      SYM(INSENSITIVE_SYM)},
   { "INSERT",		SYM(INSERT)},
   { "INSERT_METHOD",    SYM(INSERT_METHOD)},
+  { "INSTALL",          SYM(INSTALL_SYM)},
   { "INT",		SYM(INT_SYM)},
   { "INT1",		SYM(TINYINT)},
   { "INT2",		SYM(SMALLINT)},
@@ -370,6 +371,7 @@ static SYMBOL symbols[] = {
   { "OUTER",		SYM(OUTER)},
   { "OUTFILE",		SYM(OUTFILE)},
   { "PACK_KEYS",	SYM(PACK_KEYS_SYM)},
+  { "PARSER",           SYM(PARSER_SYM)},
   { "PARTIAL",		SYM(PARTIAL)},
 #ifdef HAVE_PARTITION_DB
   { "PARTITION",        SYM(PARTITION_SYM)},
@@ -377,6 +379,7 @@ static SYMBOL symbols[] = {
   { "PARTITIONS",       SYM(PARTITIONS_SYM)},
   { "PASSWORD",		SYM(PASSWORD)},
   { "PHASE",            SYM(PHASE_SYM)},
+  { "PLUGIN",           SYM(PLUGIN_SYM)},
   { "POINT",		SYM(POINT_SYM)},
   { "POLYGON",		SYM(POLYGON)},
   { "PRECISION",	SYM(PRECISION)},
@@ -454,7 +457,7 @@ static SYMBOL symbols[] = {
   { "SNAPSHOT",         SYM(SNAPSHOT_SYM)},
   { "SMALLINT",		SYM(SMALLINT)},
   { "SOME",             SYM(ANY_SYM)},
-  { "SONAME",		SYM(UDF_SONAME_SYM)},
+  { "SONAME",		SYM(SONAME_SYM)},
   { "SOUNDS",		SYM(SOUNDS_SYM)},
   { "SPATIAL",		SYM(SPATIAL_SYM)},
   { "SPECIFIC",         SYM(SPECIFIC_SYM)},
@@ -525,6 +528,7 @@ static SYMBOL symbols[] = {
   { "UNIQUE",		SYM(UNIQUE_SYM)},
   { "UNKNOWN",		SYM(UNKNOWN_SYM)},
   { "UNLOCK",		SYM(UNLOCK_SYM)},
+  { "UNINSTALL",        SYM(UNINSTALL_SYM)},
   { "UNSIGNED",		SYM(UNSIGNED)},
   { "UNTIL",		SYM(UNTIL_SYM)},
   { "UPDATE",		SYM(UPDATE_SYM)},

sql/mysqld.cc

@@ -1072,10 +1072,13 @@ void clean_up(bool print_message)
   lex_free();				/* Free some memory */
   set_var_free();
   free_charsets();
-#ifdef HAVE_DLOPEN
   if (!opt_noacl)
+  {
+#ifdef HAVE_DLOPEN
     udf_free();
 #endif
+    plugin_free();
+  }
   (void) ha_panic(HA_PANIC_CLOSE);	/* close all tables and logs */
   if (tc_log)
     tc_log->close();
@@ -3413,10 +3416,13 @@ we force server id to 2, but this MySQL server will not act as a slave.");
   if (!opt_noacl)
     (void) grant_init();
 
-#ifdef HAVE_DLOPEN
   if (!opt_noacl)
+  {
+    plugin_init();
+#ifdef HAVE_DLOPEN
     udf_init();
 #endif
+  }
   if (opt_bootstrap) /* If running with bootstrap, do not start replication. */
     opt_skip_slave_start= 1;
   /*
@@ -4574,7 +4580,8 @@ enum options_mysqld
   OPT_TIMED_MUTEXES,
   OPT_OLD_STYLE_USER_LIMITS,
   OPT_LOG_SLOW_ADMIN_STATEMENTS,
-  OPT_TABLE_LOCK_WAIT_TIMEOUT
+  OPT_TABLE_LOCK_WAIT_TIMEOUT,
+  OPT_PLUGIN_DIR
 };
 
 
@@ -5724,6 +5731,10 @@ The minimum value for this variable is 4096.",
    (gptr*) &global_system_variables.optimizer_search_depth,
    (gptr*) &max_system_variables.optimizer_search_depth,
    0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0},
+  {"plugin_dir", OPT_PLUGIN_DIR,
+   "Directory for plugins.",
+   (gptr*) &opt_plugin_dir_ptr, (gptr*) &opt_plugin_dir_ptr, 0,
+   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"preload_buffer_size", OPT_PRELOAD_BUFFER_SIZE,
     "The size of the buffer that is allocated when preloading indexes",
     (gptr*) &global_system_variables.preload_buff_size,
@@ -6292,6 +6303,9 @@ static void mysql_init_variables(void)
 	  sizeof(mysql_real_data_home)-1);
   mysql_data_home_buff[0]=FN_CURLIB;	// all paths are relative from here
   mysql_data_home_buff[1]=0;
+  strmake(opt_plugin_dir, get_relative_path(LIBDIR),
+          sizeof(opt_plugin_dir) - 1);
+  opt_plugin_dir_ptr= opt_plugin_dir;
 
   /* Replication parameters */
   master_user= (char*) "test";
@@ -7215,6 +7229,7 @@ static void fix_paths(void)
   (void) my_load_path(mysql_home,mysql_home,""); // Resolve current dir
   (void) my_load_path(mysql_real_data_home,mysql_real_data_home,mysql_home);
   (void) my_load_path(pidfile_name,pidfile_name,mysql_real_data_home);
+  (void) my_load_path(opt_plugin_dir, opt_plugin_dir_ptr, mysql_home);
 
   char *sharedir=get_relative_path(SHAREDIR);
   if (test_if_hard_path(sharedir))

sql/set_var.cc

@@ -781,6 +781,7 @@ struct show_var_st init_vars[]= {
   {sys_optimizer_search_depth.name,(char*) &sys_optimizer_search_depth,
    SHOW_SYS},
   {"pid_file",                (char*) pidfile_name,                 SHOW_CHAR},
+  {"plugin_dir",              (char*) opt_plugin_dir,               SHOW_CHAR},
   {"port",                    (char*) &mysqld_port,                  SHOW_INT},
   {sys_preload_buff_size.name, (char*) &sys_preload_buff_size,      SHOW_SYS},
   {"protocol_version",        (char*) &protocol_version,            SHOW_INT},

sql/share/errmsg.txt

@@ -3006,7 +3006,7 @@ ER_CANT_FIND_DL_ENTRY
 	cze "Nemohu naj-Bt funkci '%-.64s' v knihovn'"
 	dan "Kan ikke finde funktionen '%-.64s' i bibliotek'"
 	nla "Kan functie '%-.64s' niet in library vinden"
-	eng "Can't find function '%-.64s' in library'"
+	eng "Can't find symbol '%-.64s' in library'"
 	jps "function '%-.64s' Cu[Ɍt鎖ł܂",
 	est "Ei leia funktsiooni '%-.64s' antud teegis"
 	fre "Impossible de trouver la fonction '%-.64s' dans la bibliothque'"
@@ -3018,7 +3018,7 @@ ER_CANT_FIND_DL_ENTRY
 	kor "̹ '%-.64s' Լ ã  ϴ."
 	por "No pode encontrar a funo '%-.64s' na biblioteca"
 	rum "Nu pot gasi functia '%-.64s' in libraria"
-	rus "   '%-.64s'  "
+	rus "   '%-.64s'  "
 	serbian "Ne mogu da pronadjem funkciju '%-.64s' u biblioteci"
 	slo "Nemem njs funkciu '%-.64s' v kninici'"
 	spa "No puedo encontrar funcin '%-.64s' en libraria'"

sql/sql_class.h

@@ -413,11 +413,13 @@ public:
   List<key_part_spec> columns;
   const char *name;
   bool generated;
+  LEX_STRING *parser_name;
 
   Key(enum Keytype type_par, const char *name_arg, enum ha_key_alg alg_par,
-      bool generated_arg, List<key_part_spec> &cols)
+      bool generated_arg, List<key_part_spec> &cols,
+      LEX_STRING *parser_arg= 0)
     :type(type_par), algorithm(alg_par), columns(cols), name(name_arg),
-    generated(generated_arg)
+    generated(generated_arg), parser_name(parser_arg)
   {}
   ~Key() {}
   /* Equality comparison of keys (ignoring name) */

sql/sql_lex.h

@@ -91,6 +91,7 @@ enum enum_sql_command {
   SQLCOM_CREATE_TRIGGER, SQLCOM_DROP_TRIGGER,
   SQLCOM_XA_START, SQLCOM_XA_END, SQLCOM_XA_PREPARE,
   SQLCOM_XA_COMMIT, SQLCOM_XA_ROLLBACK, SQLCOM_XA_RECOVER,
+  SQLCOM_INSTALL_PLUGIN, SQLCOM_UNINSTALL_PLUGIN,
   /* This should be the last !!! */
 
   SQLCOM_END

sql/sql_parse.cc

@@ -4780,6 +4780,15 @@ end_with_restore_list:
   case SQLCOM_XA_RECOVER:
     res= mysql_xa_recover(thd);
     break;
+  case SQLCOM_INSTALL_PLUGIN:
+    if (! (res= mysql_install_plugin(thd, &thd->lex->comment,
+                                     &thd->lex->ident)))
+      send_ok(thd);
+    break;
+  case SQLCOM_UNINSTALL_PLUGIN:
+    if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment)))
+      send_ok(thd);
+    break;
   default:
     DBUG_ASSERT(0);                             /* Impossible */
     send_ok(thd);

sql/sql_show.cc

@@ -948,6 +948,12 @@ store_create_info(THD *thd, TABLE_LIST *table_list, String *packet)
       }
     }
     packet->append(')');
+    if (key_info->parser)
+    {
+      packet->append(" WITH PARSER ", 13);
+      append_identifier(thd, packet, key_info->parser->name.str,
+                        key_info->parser->name.length);
+    }
   }
 
   /*

sql/sql_table.cc

@@ -1087,6 +1087,8 @@ static int mysql_prepare_table(THD *thd, HA_CREATE_INFO *create_info,
 	break;
     case Key::FULLTEXT:
 	key_info->flags= HA_FULLTEXT;
+	if ((key_info->parser_name= key->parser_name))
+          key_info->flags|= HA_USES_PARSER;
 	break;
     case Key::SPATIAL:
 #ifdef HAVE_SPATIAL

sql/sql_udf.cc

@@ -38,36 +38,10 @@
 #ifdef HAVE_DLOPEN
 extern "C"
 {
-#if defined(__WIN__)
-  void* dlsym(void* lib,const char* name)
-  {
-    return GetProcAddress((HMODULE)lib,name);
-  }
-  void* dlopen(const char* libname,int unused)
-  {
-    return LoadLibraryEx(libname,NULL,0);
-  }
-  void dlclose(void* lib)
-  {
-    FreeLibrary((HMODULE)lib);
-  }
-
-#elif !defined(OS2)
-#include <dlfcn.h>
-#endif
-
 #include <stdarg.h>
 #include <hash.h>
 }
 
-#ifndef RTLD_NOW
-#define RTLD_NOW 1				// For FreeBSD 2.2.2
-#endif
-
-#ifndef HAVE_DLERROR
-#define dlerror() ""
-#endif
-
 static bool initialized = 0;
 static MEM_ROOT mem;
 static HASH udf_hash;
@@ -194,8 +168,10 @@ void udf_init()
       This is done to ensure that only approved dll from the system
       directories are used (to make this even remotely secure).
     */
-    if (strchr(dl_name, '/') ||
-        IF_WIN(strchr(dl_name, '\\'),0) ||
+    if (my_strchr(files_charset_info, dl_name,
+                  dl_name + strlen(dl_name), '/') ||
+        IF_WIN(my_strchr(files_charset_info, dl_name,
+                         dl_name + strlen(dl_name), '\\'),0) ||
         strlen(name.str) > NAME_LEN)
     {
       sql_print_error("Invalid row in mysql.func table for function '%.64s'",
@@ -214,10 +190,13 @@ void udf_init()
     void *dl = find_udf_dl(tmp->dl);
     if (dl == NULL)
     {
-      if (!(dl = dlopen(tmp->dl, RTLD_NOW)))
+      char dlpath[FN_REFLEN];
+      strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", tmp->dl,
+               NullS);
+      if (!(dl= dlopen(dlpath, RTLD_NOW)))
       {
 	/* Print warning to log */
-	sql_print_error(ER(ER_CANT_OPEN_LIBRARY), tmp->dl,errno,dlerror());
+	sql_print_error(ER(ER_CANT_OPEN_LIBRARY), dlpath, errno, dlerror());
 	/* Keep the udf in the hash so that we can remove it later */
 	continue;
       }
@@ -412,7 +391,9 @@ int mysql_create_function(THD *thd,udf_func *udf)
     This is done to ensure that only approved dll from the system
     directories are used (to make this even remotely secure).
   */
-  if (strchr(udf->dl, '/') || IF_WIN(strchr(udf->dl, '\\'),0))
+  if (my_strchr(files_charset_info, udf->dl, udf->dl + strlen(udf->dl), '/') ||
+      IF_WIN(strchr(files_charset_info, udf->dl,
+                    udf->dl + strlen(udf->dl), '\\'),0))
   {
     my_message(ER_UDF_NO_PATHS, ER(ER_UDF_NO_PATHS), MYF(0));
     DBUG_RETURN(1);
@@ -431,12 +412,14 @@ int mysql_create_function(THD *thd,udf_func *udf)
   }
   if (!(dl = find_udf_dl(udf->dl)))
   {
-    if (!(dl = dlopen(udf->dl, RTLD_NOW)))
+    char dlpath[FN_REFLEN];
+    strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", udf->dl, NullS);
+    if (!(dl = dlopen(dlpath, RTLD_NOW)))
     {
       DBUG_PRINT("error",("dlopen of %s failed, error: %d (%s)",
-			  udf->dl,errno,dlerror()));
+			  dlpath, errno, dlerror()));
       my_error(ER_CANT_OPEN_LIBRARY, MYF(0),
-                      udf->dl, errno, dlerror());
+                      dlpath, errno, dlerror());
       goto err;
     }
     new_dl=1;

sql/sql_yacc.yy

@@ -335,6 +335,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  INSENSITIVE_SYM
 %token  INSERT
 %token  INSERT_METHOD
+%token  INSTALL_SYM
 %token  INTERVAL_SYM
 %token  INTO
 %token  INT_SYM
@@ -470,12 +471,14 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  OUTFILE
 %token  OUT_SYM
 %token  PACK_KEYS_SYM
+%token  PARSER_SYM
 %token  PARTIAL
 %token  PARTITION_SYM
 %token  PARTITIONS_SYM
 %token  PASSWORD
 %token  PARAM_MARKER
 %token  PHASE_SYM
+%token  PLUGIN_SYM
 %token  POINTFROMTEXT
 %token  POINT_SYM
 %token  POLYFROMTEXT
@@ -560,6 +563,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  SLAVE
 %token  SMALLINT
 %token  SNAPSHOT_SYM
+%token  SONAME_SYM
 %token  SOUNDS_SYM
 %token  SPATIAL_SYM
 %token  SPECIFIC_SYM
@@ -621,13 +625,13 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  TYPES_SYM
 %token  TYPE_SYM
 %token  UDF_RETURNS_SYM
-%token  UDF_SONAME_SYM
 %token  ULONGLONG_NUM
 %token  UNCOMMITTED_SYM
 %token  UNDEFINED_SYM
 %token  UNDERSCORE_CHARSET
 %token  UNDO_SYM
 %token  UNICODE_SYM
+%token  UNINSTALL_SYM
 %token  UNION_SYM
 %token  UNIQUE_SYM
 %token  UNIQUE_USERS
@@ -696,7 +700,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
         sp_opt_label BIN_NUM label_ident
 
 %type <lex_str_ptr>
-	opt_table_alias
+	opt_table_alias opt_fulltext_parser
 
 %type <table>
 	table_ident table_ident_nodb references xid
@@ -844,7 +848,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 	statement sp_suid opt_view_list view_list or_replace algorithm
 	sp_c_chistics sp_a_chistics sp_chistic sp_c_chistic xa
         load_data opt_field_or_var_spec fields_or_vars opt_load_data_set_spec
-        view_user view_suid
+        install uninstall view_user view_suid
         partition_entry
 END_OF_INPUT
 
@@ -906,6 +910,7 @@ statement:
 	| handler
 	| help
 	| insert
+        | install
 	| kill
 	| load
 	| lock
@@ -930,6 +935,7 @@ statement:
 	| slave
 	| start
 	| truncate
+        | uninstall
 	| unlock
 	| update
 	| use
@@ -1187,11 +1193,15 @@ create:
 	    lex->col_list.empty();
 	    lex->change=NullS;
 	  }
-	   '(' key_list ')'
+	   '(' key_list ')' opt_fulltext_parser
 	  {
 	    LEX *lex=Lex;
-
-	    lex->key_list.push_back(new Key($2,$4.str, $5, 0, lex->col_list));
+	    if ($2 != Key::FULLTEXT && $12)
+	    {
+	      yyerror(ER(ER_SYNTAX_ERROR));
+	      YYABORT;
+	    }
+	    lex->key_list.push_back(new Key($2,$4.str,$5,0,lex->col_list,$12));
 	    lex->col_list.empty();
 	  }
 	| CREATE DATABASE opt_if_not_exists ident
@@ -1382,7 +1392,7 @@ sp_name:
 	;
 
 create_function_tail:
-	  RETURNS_SYM udf_type UDF_SONAME_SYM TEXT_STRING_sys
+	  RETURNS_SYM udf_type SONAME_SYM TEXT_STRING_sys
 	  {
 	    LEX *lex=Lex;
 	    lex->sql_command = SQLCOM_CREATE_FUNCTION;
@@ -3302,10 +3312,15 @@ column_def:
 	;
 
 key_def:
-	key_type opt_ident key_alg '(' key_list ')'
+	key_type opt_ident key_alg '(' key_list ')' opt_fulltext_parser
 	  {
 	    LEX *lex=Lex;
-	    lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list));
+	    if ($1 != Key::FULLTEXT && $7)
+	    {
+	      yyerror(ER(ER_SYNTAX_ERROR));
+	      YYABORT;
+	    }
+	    lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list, $7));
 	    lex->col_list.empty();		/* Alloced by sql_alloc */
 	  }
 	| opt_constraint constraint_key_type opt_ident key_alg '(' key_list ')'
@@ -3340,6 +3355,20 @@ key_def:
 	  }
 	;
 
+opt_fulltext_parser:
+        /* empty */               { $$= (LEX_STRING *)0; }
+	| WITH PARSER_SYM IDENT_sys
+          {
+            if (plugin_is_ready(&$3, MYSQL_FTPARSER_PLUGIN))
+              $$= (LEX_STRING *)sql_memdup(&$3, sizeof(LEX_STRING));
+            else
+            {
+              my_error(ER_FUNCTION_NOT_DEFINED, MYF(0), $3.str);
+              YYABORT;
+            }
+          }
+        ;
+
 opt_check_constraint:
 	/* empty */
 	| check_constraint
@@ -8153,10 +8182,13 @@ keyword:
 	| FLUSH_SYM		{}
 	| HANDLER_SYM		{}
 	| HELP_SYM		{}
+        | INSTALL_SYM           {}
 	| LANGUAGE_SYM          {}
 	| NO_SYM		{}
 	| OPEN_SYM		{}
+        | PARSER_SYM            {}
 	| PARTITION_SYM		{}
+        | PLUGIN_SYM            {}
         | PREPARE_SYM           {}
 	| REPAIR		{}
 	| RESET_SYM		{}
@@ -8166,10 +8198,12 @@ keyword:
 	| SECURITY_SYM		{}
 	| SIGNED_SYM		{}
 	| SLAVE			{}
+        | SONAME_SYM            {}
 	| START_SYM		{}
 	| STOP_SYM		{}
 	| TRUNCATE_SYM		{}
 	| UNICODE_SYM		{}
+        | UNINSTALL_SYM         {}
         | XA_SYM                {}
 	;
 
@@ -9732,4 +9766,19 @@ opt_migrate:
     | FOR_SYM MIGRATE_SYM   { Lex->xa_opt=XA_FOR_MIGRATE; }
     ;
 
+install:
+  INSTALL_SYM PLUGIN_SYM IDENT_sys SONAME_SYM TEXT_STRING_sys
+  {
+    LEX *lex= Lex;
+    lex->sql_command= SQLCOM_INSTALL_PLUGIN;
+    lex->comment= $3;
+    lex->ident= $5;
+  };
 
+uninstall:
+  UNINSTALL_SYM PLUGIN_SYM IDENT_sys
+  {
+    LEX *lex= Lex;
+    lex->sql_command= SQLCOM_UNINSTALL_PLUGIN;
+    lex->comment= $3;
+  };

sql/table.cc

@@ -323,6 +323,7 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
     if (! (share->connect_string.str= strmake_root(&outparam->mem_root,
             next_chunk + 2, share->connect_string.length)))
     {
+      DBUG_PRINT("EDS", ("strmake_root failed for connect_string"));
       my_free(buff, MYF(0));
       goto err;
     }
@@ -1064,9 +1065,17 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
 int closefrm(register TABLE *table)
 {
   int error=0;
+  uint idx;
+  KEY *key_info;
   DBUG_ENTER("closefrm");
   if (table->db_stat)
     error=table->file->close();
+  key_info= table->key_info;
+  for (idx= table->s->keys; idx; idx--, key_info++)
+  {
+    if (key_info->flags & HA_USES_PARSER)
+      plugin_unlock(key_info->parser);
+  }
   my_free((char*) table->alias, MYF(MY_ALLOW_ZERO_PTR));
   table->alias= 0;
   if (table->field)

storage/myisam/ft_nlq_search.c

@@ -230,7 +230,7 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
             NULL, NULL);
 
   ft_parse_init(&wtree, aio.charset);
-  if (ft_parse(&wtree,query,query_len,0))
+  if (ft_parse(&wtree, query, query_len, 0, info->s->keyinfo[keynr].parser))
     goto err;
 
   if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,

storage/myisam/ft_parser.c

@@ -24,6 +24,14 @@ typedef struct st_ft_docstat {
   double sum;
 } FT_DOCSTAT;
 
+
+typedef struct st_my_ft_parser_param
+{
+  TREE *wtree;
+  my_bool with_alloc;
+} MY_FT_PARSER_PARAM;
+
+
 static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
 {
   return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
@@ -102,13 +110,14 @@ my_bool ft_boolean_check_syntax_string(const byte *str)
   4 - stopword found
 */
 byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
-                 FT_WORD *word, FTB_PARAM *param)
+                 FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
 {
   byte *doc=*start;
   uint mwc, length, mbl;
 
   param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
-  param->plusminus=param->pmsign=0;
+  param->weight_adjust= param->wasign= 0;
+  param->type= FT_TOKEN_EOF;
 
   while (doc<end)
   {
@@ -119,7 +128,8 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
       {
         param->quot=doc;
         *start=doc+1;
-        return 3; /* FTB_RBR */
+        param->type= FT_TOKEN_RIGHT_PAREN;
+        goto ret;
       }
       if (!param->quot)
       {
@@ -128,21 +138,22 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
           /* param->prev=' '; */
           *start=doc+1;
           if (*doc == FTB_LQUOT) param->quot=*start;
-          return (*doc == FTB_RBR)+2;
+          param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
+          goto ret;
         }
         if (param->prev == ' ')
         {
           if (*doc == FTB_YES ) { param->yesno=+1;    continue; } else
           if (*doc == FTB_EGAL) { param->yesno= 0;    continue; } else
           if (*doc == FTB_NO  ) { param->yesno=-1;    continue; } else
-          if (*doc == FTB_INC ) { param->plusminus++; continue; } else
-          if (*doc == FTB_DEC ) { param->plusminus--; continue; } else
-          if (*doc == FTB_NEG ) { param->pmsign=!param->pmsign; continue; }
+          if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
+          if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
+          if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
         }
       }
       param->prev=*doc;
       param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
-      param->plusminus=param->pmsign=0;
+      param->weight_adjust= param->wasign= 0;
     }
 
     mwc=length=0;
@@ -161,20 +172,24 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
          || param->trunc) && length < ft_max_word_len)
     {
       *start=doc;
-      return 1;
+      param->type= FT_TOKEN_WORD;
+      goto ret;
     }
     else if (length) /* make sure length > 0 (if start contains spaces only) */
     {
       *start= doc;
-      return 4;
+      param->type= FT_TOKEN_STOPWORD;
+      goto ret;
     }
   }
   if (param->quot)
   {
     param->quot=*start=doc;
-    return 3; /* FTB_RBR */
+    param->type= 3; /* FT_RBR */
+    goto ret;
   }
-  return 0;
+ret:
+  return param->type;
 }
 
 byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end,
@@ -220,30 +235,67 @@ void ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
   DBUG_VOID_RETURN;
 }
 
-int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc)
+
+static int ft_add_word(void *param, byte *word, uint word_len,
+             MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
 {
-  byte   *end=doc+doclen;
+  TREE *wtree;
   FT_WORD w;
-  DBUG_ENTER("ft_parse");
-
-  while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
+  DBUG_ENTER("ft_add_word");
+  wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
+  if (((MY_FT_PARSER_PARAM *)param)->with_alloc)
   {
-    if (with_alloc)
-    {
-      byte *ptr;
-      /* allocating the data in the tree - to avoid mallocs and frees */
-      DBUG_ASSERT(wtree->with_delete==0);
-      ptr=(byte *)alloc_root(& wtree->mem_root,w.len);
-      memcpy(ptr, w.pos, w.len);
-      w.pos=ptr;
-    }
-    if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
-      goto err;
+    byte *ptr;
+    /* allocating the data in the tree - to avoid mallocs and frees */
+    DBUG_ASSERT(wtree->with_delete == 0);
+    ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
+    memcpy(ptr, word, word_len);
+    w.pos= ptr;
+  }
+  else
+    w.pos= word;
+  w.len= word_len;
+  if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
+  {
+    delete_tree(wtree);
+    DBUG_RETURN(1);
   }
   DBUG_RETURN(0);
+}
 
-err:
-  delete_tree(wtree);
-  DBUG_RETURN(1);
+
+static int ft_parse_internal(void *param, byte *doc, uint doc_len)
+{
+  byte   *end=doc+doc_len;
+  FT_WORD w;
+  TREE *wtree;
+  DBUG_ENTER("ft_parse_internal");
+
+  wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
+  while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
+    if (ft_add_word(param, w.pos, w.len, 0))
+      DBUG_RETURN(1);
+  DBUG_RETURN(0);
 }
 
+
+int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
+                    struct st_mysql_ftparser *parser)
+{
+  MYSQL_FTPARSER_PARAM param;
+  MY_FT_PARSER_PARAM my_param;
+  DBUG_ENTER("ft_parse");
+  DBUG_ASSERT(parser);
+  my_param.wtree= wtree;
+  my_param.with_alloc= with_alloc;
+
+  param.mysql_parse= ft_parse_internal;
+  param.mysql_add_word= ft_add_word;
+  param.ftparser_state= 0;
+  param.mysql_ftparam= &my_param;
+  param.cs= wtree->custom_arg;
+  param.doc= doc;
+  param.length= doclen;
+  param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
+  DBUG_RETURN(parser->parse(&param));
+}

storage/myisam/ft_static.c

@@ -626,3 +626,14 @@ const char *ft_precompiled_stopwords[] = {
 #endif
 
   NULL };
+
+static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param)
+{
+  return param->mysql_parse(param->mysql_ftparam, param->doc, param->length);
+}
+
+struct st_mysql_ftparser ft_default_parser=
+{
+  MYSQL_FTPARSER_INTERFACE_VERSION, ft_default_parser_parse, 0, 0
+};
+

storage/myisam/ft_update.c

@@ -99,15 +99,17 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
                   const byte *record, my_bool with_alloc)
 {
   FT_SEG_ITERATOR ftsi;
+  struct st_mysql_ftparser *parser;
   DBUG_ENTER("_mi_ft_parse");
 
   _mi_ft_segiterator_init(info, keynr, record, &ftsi);
 
   ft_parse_init(parsed, info->s->keyinfo[keynr].seg->charset);
+  parser= info->s->keyinfo[keynr].parser;
   while (_mi_ft_segiterator(&ftsi))
   {
     if (ftsi.pos)
-      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc))
+      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser))
         DBUG_RETURN(1);
   }
   DBUG_RETURN(0);

storage/myisam/ftdefs.h

@@ -22,6 +22,7 @@
 #include <m_ctype.h>
 #include <my_tree.h>
 #include <queues.h>
+#include <plugin.h>
 
 #define true_word_char(s,X)	(my_isalnum(s,X) || (X)=='_')
 #define misc_word_char(X)	((X)=='\'')
@@ -98,20 +99,12 @@ typedef struct st_ft_word {
   double weight;
 } FT_WORD;
 
-typedef struct st_ftb_param {
-  byte prev;
-  int  yesno;
-  int  plusminus;
-  bool pmsign;
-  bool trunc;
-  byte *quot;
-} FTB_PARAM;
-
 int is_stopword(char *word, uint len);
 
 uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t);
 
-byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *, FTB_PARAM *);
+byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *,
+                 MYSQL_FTPARSER_BOOLEAN_INFO *);
 byte ft_simple_get_word(CHARSET_INFO *, byte **, const byte *,
                         FT_WORD *, my_bool);
 
@@ -126,7 +119,7 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
 uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
 
 void ft_parse_init(TREE *, CHARSET_INFO *);
-int ft_parse(TREE *, byte *, int, my_bool);
+int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser);
 FT_WORD * ft_linearize(TREE *);
 FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
 uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool);

storage/myisam/mi_open.c

@@ -1047,6 +1047,7 @@ char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef)
    keydef->block_size	= keydef->block_length/MI_MIN_KEY_BLOCK_LENGTH-1;
    keydef->underflow_block_length=keydef->block_length/3;
    keydef->version	= 0;			/* Not saved */
+   keydef->parser       = &ft_default_parser;
    return ptr;
 }