MDEV-5549 Wrong row counter in found_rows() result

only let filesort() count rows for SQL_CALC_ROWS if it's using priority queue

mysql-test/r/select_found.result

@@ -280,3 +280,14 @@ SELECT FOUND_ROWS();
 FOUND_ROWS()
 1
 DROP TABLE t1;
+create table t1 (f1 int primary key, f2 tinyint) engine=myisam;
+insert t1 values (10,3),(11,2),(12,3);
+create table t2 (f3 int primary key) engine=myisam;
+insert t2 values (11),(12),(13);
+select f1 from t1,t2 where f1=f3 and f2=3 order by f1;
+f1
+12
+select found_rows();
+found_rows()
+1
+drop table t1, t2;

mysql-test/t/select_found.test

@@ -195,3 +195,16 @@ SELECT FOUND_ROWS();
 DROP TABLE t1;
 
 # End of 4.1 tests
+
+#
+# MDEV-5549 Wrong row counter in found_rows() result
+#
+create table t1 (f1 int primary key, f2 tinyint) engine=myisam;
+insert t1 values (10,3),(11,2),(12,3);
+create table t2 (f3 int primary key) engine=myisam;
+insert t2 values (11),(12),(13);
+#explain select f1 from t1,t2 where f1=f3 and f2=3 order by f1;
+select f1 from t1,t2 where f1=f3 and f2=3 order by f1;
+select found_rows();
+drop table t1, t2;
+

sql/filesort.cc

@@ -595,7 +595,7 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
   ref_pos= ref_buff;
   quick_select=select && select->quick;
   record=0;
-  *found_rows= 0;
+  *found_rows= pq ? 0 : HA_POS_ERROR; // don't count unless pq is used
   flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select);
   if (flag)
     ref_pos= &file->ref[0];
@@ -714,9 +714,14 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
 
     if (write_record)
     {
-       ++(*found_rows);
       if (pq)
       {
+        /*
+          only count rows when pq is used - otherwise there might be
+          other filters *after* the filesort, we don't know the final row
+          count here
+        */
+        (*found_rows)++;
         pq->push(ref_pos);
         idx= pq->num_elements();
       }

sql/sql_select.cc

@@ -3065,8 +3065,7 @@ void JOIN::exec_inner()
                                    Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
   error= do_select(curr_join, curr_fields_list, NULL, procedure);
   thd->limit_found_rows= curr_join->send_records;
-  if (curr_join->order &&
-      curr_join->sortorder)
+  if (curr_join->order && curr_join->filesort_found_rows)
   {
     /* Use info provided by filesort. */
     DBUG_ASSERT(curr_join->table_count > curr_join->const_tables);
@@ -18535,8 +18534,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         records are read. Because of optimization in some cases it can
         provide only select_limit_cnt+1 records.
       */
-      if (join->order &&
-          join->sortorder &&
+      if (join->order && join->filesort_found_rows &&
           join->select_options & OPTION_FOUND_ROWS)
       {
         DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT"));
@@ -20394,7 +20392,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order,
                             select, filesort_limit, 0,
                             &examined_rows, &found_rows);
   table->sort.found_records= filesort_retval;
-  tab->records= found_rows;                     // For SQL_CALC_ROWS
+  if (found_rows != HA_POS_ERROR)
+  {
+    tab->records= found_rows;                     // For SQL_CALC_ROWS
+    join->filesort_found_rows= true;
+  }
 
   if (quick_created)
   {

sql/sql_select.h

@@ -1119,6 +1119,12 @@ public:
     restore_no_rows_in_result() in ::reinit()
   */
   bool no_rows_in_result_called;
+
+  /**
+    This is set if SQL_CALC_ROWS was calculated by filesort()
+    and should be taken from the appropriate JOIN_TAB
+  */
+  bool filesort_found_rows;
   
   /**
     Copy of this JOIN to be used with temporary tables.
@@ -1334,6 +1340,7 @@ public:
     emb_sjm_nest= NULL;
     sjm_lookup_tables= 0;
 
+    filesort_found_rows= false;
     exec_saved_explain= false;
     /* 
       The following is needed because JOIN::cleanup(true) may be called for