BUG# 9148: Denial of service

This is a second patch needing another review.  The first patch didn't solve
the entire problem.  open and fopen on Windows will still open
files like "com1.sym" when they shouldn't.  This patch
checks that the file exists before trying to open it.

mysys/my_fopen.c

@@ -33,9 +33,21 @@ FILE *my_fopen(const char *FileName, int Flags, myf MyFlags)
   DBUG_ENTER("my_fopen");
   DBUG_PRINT("my",("Name: '%s'  Flags: %d  MyFlags: %d",
 		   FileName, Flags, MyFlags));
-
-  make_ftype(type,Flags);
-  if ((fd = fopen(FileName, type)) != 0)
+  /* 
+   * if we are not creating, then we need to use my_access to make sure  
+   * the file exists since Windows doesn't handle files like "com1.sym" very  well 
+  */
+#ifdef __WIN__
+  if (! (Flags & O_CREAT) && my_access(FileName, F_OK))
+	  fd=0;
+  else
+#endif
+  {
+  	make_ftype(type,Flags);
+  	fd = fopen(FileName, type);
+  }
+  
+  if (fd != 0)
   {
     /*
       The test works if MY_NFILE < 128. The problem is that fileno() is char

mysys/my_open.c

@@ -46,6 +46,12 @@ File my_open(const char *FileName, int Flags, myf MyFlags)
   DBUG_PRINT("my",("Name: '%s'  Flags: %d  MyFlags: %d",
 		   FileName, Flags, MyFlags));
 #if defined(MSDOS) || defined(__WIN__) || defined(__EMX__) || defined(OS2)
+  /* if we are not creating, then we need to use my_access to make 
+   * sure  the file exists since Windows doesn't handle files like 
+   * "com1.sym" very  well 
+  */  
+  if (! (Flags & O_CREAT) && my_access(FileName, F_OK))    
+	  return -1;
   if (Flags & O_SHARE)
     fd = sopen((my_string) FileName, (Flags & ~O_SHARE) | O_BINARY, SH_DENYNO,
 	       MY_S_IREAD | MY_S_IWRITE);