MDEV-9256 : Crashes on Windows x64 with aria_pagecache_buffer_size > 4GB

Fixed wrong calculation of buffer sizes. ulong datatype was used wrongly,
as were the casts to ulong. Buffer sizes should be of type size_t,
not ulong, or bad things happen on 64 bit Windows.

This patch changes pagecache struct to use size_t/ssize_t
where long/ulong were previously used. Also, removed several casts.

storage/maria/ma_pagecache.c

@@ -500,8 +500,8 @@ static void test_key_cache(PAGECACHE *pagecache,
                            const char *where, my_bool lock);
 #endif
 
-#define PAGECACHE_HASH(p, f, pos) (((ulong) (pos) +                          \
-                                    (ulong) (f).file) & (p->hash_entries-1))
+#define PAGECACHE_HASH(p, f, pos) (((size_t) (pos) +                          \
+                                    (size_t) (f).file) & (p->hash_entries-1))
 #define FILE_HASH(f) ((uint) (f).file & (PAGECACHE_CHANGED_BLOCKS_HASH - 1))
 
 #define DEFAULT_PAGECACHE_DEBUG_LOG  "pagecache_debug.log"
@@ -639,10 +639,10 @@ static my_bool pagecache_fwrite(PAGECACHE *pagecache,
   {
     char buff[80];
     uint len= my_sprintf(buff,
-                         (buff, "fwrite: fd: %d  id: %u  page: %lu",
+                         (buff, "fwrite: fd: %d  id: %u  page: %llu",
                           filedesc->file,
                           _ma_file_callback_to_id(filedesc->callback_data),
-                          (ulong) pageno));
+                          pageno));
     (void) translog_log_debug_info(0, LOGREC_DEBUG_INFO_QUERY,
                                    (uchar*) buff, len);
   }
@@ -741,11 +741,11 @@ static inline uint next_power(uint value)
 
 */
 
-ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
+size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem,
                      uint division_limit, uint age_threshold,
                      uint block_size, myf my_readwrite_flags)
 {
-  ulong blocks, hash_links, length;
+  size_t blocks, hash_links, length;
   int error;
   DBUG_ENTER("init_pagecache");
   DBUG_ASSERT(block_size >= 512);
@@ -782,10 +782,10 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
   DBUG_PRINT("info", ("block_size: %u", block_size));
   DBUG_ASSERT(((uint)(1 << pagecache->shift)) == block_size);
 
-  blocks= (ulong) (use_mem / (sizeof(PAGECACHE_BLOCK_LINK) +
+  blocks= use_mem / (sizeof(PAGECACHE_BLOCK_LINK) +
                               2 * sizeof(PAGECACHE_HASH_LINK) +
                               sizeof(PAGECACHE_HASH_LINK*) *
-                              5/4 + block_size));
+                              5/4 + block_size);
   /*
     We need to support page cache with just one block to be able to do
     scanning of rows-in-block files
@@ -816,7 +816,7 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
       blocks--;
     /* Allocate memory for cache page buffers */
     if ((pagecache->block_mem=
-         my_large_malloc((ulong) blocks * pagecache->block_size,
+      my_large_malloc(blocks * pagecache->block_size,
                          MYF(MY_WME))))
     {
       /*
@@ -824,7 +824,7 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
         For each block 2 hash links are allocated
       */
       if ((pagecache->block_root=
-           (PAGECACHE_BLOCK_LINK*) my_malloc((size_t) length, MYF(0))))
+           (PAGECACHE_BLOCK_LINK*) my_malloc(length, MYF(0))))
         break;
       my_large_free(pagecache->block_mem);
       pagecache->block_mem= 0;
@@ -832,7 +832,7 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
     blocks= blocks / 4*3;
   }
   pagecache->blocks_unused= blocks;
-  pagecache->disk_blocks= (long) blocks;
+  pagecache->disk_blocks= blocks;
   pagecache->hash_links= hash_links;
   pagecache->hash_root=
     (PAGECACHE_HASH_LINK**) ((char*) pagecache->block_root +
@@ -887,7 +887,7 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
         PAGECACHE_CHANGED_BLOCKS_HASH);
 
   pagecache->blocks= pagecache->disk_blocks > 0 ? pagecache->disk_blocks : 0;
-  DBUG_RETURN((ulong) pagecache->disk_blocks);
+  DBUG_RETURN((size_t)pagecache->disk_blocks);
 
 err:
   error= my_errno;
@@ -978,11 +978,11 @@ static int flush_all_key_blocks(PAGECACHE *pagecache)
      So we disable it for now.
 */
 #if NOT_USED /* keep disabled until code is fixed see above !! */
-ulong resize_pagecache(PAGECACHE *pagecache,
+size_t resize_pagecache(PAGECACHE *pagecache,
                        size_t use_mem, uint division_limit,
                        uint age_threshold)
 {
-  ulong blocks;
+  size_t blocks;
   struct st_my_thread_var *thread;
   WQUEUE *wqueue;
 
@@ -1379,7 +1379,7 @@ static void link_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block,
                       ("linked block: %u:%1u  status: %x  #requests: %u  #available: %u",
                        PCBLOCK_NUMBER(pagecache, block), at_end, block->status,
                        block->requests, pagecache->blocks_available));
-  KEYCACHE_DBUG_ASSERT((ulong) pagecache->blocks_available <=
+  KEYCACHE_DBUG_ASSERT(pagecache->blocks_available <=
                        pagecache->blocks_used);
 #endif
   DBUG_VOID_RETURN;
@@ -2018,7 +2018,7 @@ restart:
           /* There are some never used blocks, take first of them */
           block= &pagecache->block_root[pagecache->blocks_used];
           block->buffer= ADD_TO_PTR(pagecache->block_mem,
-                                    ((ulong) pagecache->blocks_used*
+                                    (pagecache->blocks_used*
                                      pagecache->block_size),
                                     uchar*);
           pagecache->blocks_used++;
@@ -4870,7 +4870,7 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache,
                                                   LSN *min_rec_lsn)
 {
   my_bool error= 0;
-  ulong stored_list_size= 0;
+  size_t stored_list_size= 0;
   uint file_hash;
   char *ptr;
   LSN minimum_rec_lsn= LSN_MAX;

storage/maria/ma_pagecache.h

@@ -117,20 +117,20 @@ typedef struct st_pagecache_hash_link PAGECACHE_HASH_LINK;
 typedef struct st_pagecache
 {
   size_t mem_size;               /* specified size of the cache memory       */
-  ulong min_warm_blocks;         /* min number of warm blocks;               */
-  ulong age_threshold;           /* age threshold for hot blocks             */
+  size_t min_warm_blocks;        /* min number of warm blocks;               */
+  size_t age_threshold;          /* age threshold for hot blocks             */
   ulonglong time;                /* total number of block link operations    */
-  ulong hash_entries;            /* max number of entries in the hash table  */
-  long hash_links;               /* max number of hash links                 */
-  long hash_links_used;   /* number of hash links taken from free links pool */
-  long disk_blocks;              /* max number of blocks in the cache        */
-  ulong blocks_used;           /* maximum number of concurrently used blocks */
-  ulong blocks_unused;           /* number of currently unused blocks        */
-  ulong blocks_changed;          /* number of currently dirty blocks         */
-  ulong warm_blocks;             /* number of blocks in warm sub-chain       */
-  ulong cnt_for_resize_op;       /* counter to block resize operation        */
-  ulong blocks_available;     /* number of blocks available in the LRU chain */
-  long blocks;                   /* max number of blocks in the cache        */
+  size_t hash_entries;           /* max number of entries in the hash table  */
+  ssize_t hash_links;            /* max number of hash links                 */
+  ssize_t hash_links_used;       /* number of hash links taken from free links pool */
+  ssize_t disk_blocks;           /* max number of blocks in the cache        */
+  size_t blocks_used;            /* maximum number of concurrently used blocks */
+  size_t blocks_unused;          /* number of currently unused blocks        */
+  size_t blocks_changed;         /* number of currently dirty blocks         */
+  size_t warm_blocks;            /* number of blocks in warm sub-chain       */
+  size_t cnt_for_resize_op;      /* counter to block resize operation        */
+  size_t blocks_available;       /* number of blocks available in the LRU chain */
+  ssize_t blocks;                /* max number of blocks in the cache        */
   uint32 block_size;             /* size of the page buffer of a cache block */
   PAGECACHE_HASH_LINK **hash_root;/* arr. of entries into hash table buckets */
   PAGECACHE_HASH_LINK *hash_link_root;/* memory for hash table links         */
@@ -155,12 +155,12 @@ typedef struct st_pagecache
   */
 
   ulonglong param_buff_size;    /* size the memory allocated for the cache  */
-  ulong param_block_size;       /* size of the blocks in the key cache      */
-  ulong param_division_limit;   /* min. percentage of warm blocks           */
-  ulong param_age_threshold;    /* determines when hot block is downgraded  */
+  size_t param_block_size;       /* size of the blocks in the key cache      */
+  size_t param_division_limit;   /* min. percentage of warm blocks           */
+  size_t param_age_threshold;    /* determines when hot block is downgraded  */
 
   /* Statistics variables. These are reset in reset_pagecache_counters().    */
-  ulong global_blocks_changed;	/* number of currently dirty blocks          */
+  size_t global_blocks_changed;	/* number of currently dirty blocks          */
   ulonglong global_cache_w_requests;/* number of write requests (write hits) */
   ulonglong global_cache_write;     /* number of writes from cache to files  */
   ulonglong global_cache_r_requests;/* number of read requests (read hits)   */
@@ -193,10 +193,10 @@ typedef enum pagecache_flush_filter_result
 /* The default key cache */
 extern PAGECACHE dflt_pagecache_var, *dflt_pagecache;
 
-extern ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem,
+extern size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem,
                             uint division_limit, uint age_threshold,
                             uint block_size, myf my_read_flags);
-extern ulong resize_pagecache(PAGECACHE *pagecache,
+extern size_t resize_pagecache(PAGECACHE *pagecache,
                               size_t use_mem, uint division_limit,
                               uint age_threshold);
 extern void change_pagecache_param(PAGECACHE *pagecache, uint division_limit,

storage/maria/ma_sort.c

@@ -519,7 +519,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
 {
   MARIA_SORT_INFO *sort_info=sort_param->sort_info;
   HA_CHECK *param=sort_info->param;
-  ulong UNINIT_VAR(length), keys;
+  size_t UNINIT_VAR(length), keys;
   double *rec_per_key_part= param->new_rec_per_key_part;
   int got_error=sort_info->got_error;
   uint i;