MDEV-7046: MySQL#74480 - Failing assertion: os_file_status(newpath, &exists, &type)

after Operating system error number 36 in a file operation.

Analysis: os_file_get_status did not handle error ENAMETOOLONG
correctly.

Fix: Add correct handling for error ENAMETOOLONG. Note that on InnoDB
case the error is not passed all the way up to server. That would
be bigger rewamp.

mysql-test/suite/innodb/r/innodb-mdev7046.result

+call mtr.add_suppression("InnoDB: File ./test/t1*");
+call mtr.add_suppression("nnoDB: Error number*");
+USE test;
+create table t1(f1 INT,f2 INT,f3 CHAR (10),primary key(f1,f2)) partition by range(f1) subpartition by hash(f2) subpartitions 2 (partition p1 values less than (0),partition p2 values less than (2),partition p3 values less than (2147483647));
+RENAME TABLE t1 TO `t2_new..............................................end`;
+ERROR HY000: Error on rename of 't2_new' (Errcode: 36 "File name too long")
+alter table t1 engine=innodb;
+RENAME TABLE t1 TO `t2_new..............................................end`;
+ERROR HY000: Error on rename of 't2_new' (errno: -1 "Internal error < 0 (Not system error)")
+show warnings;
+Level	Code	Message
+Error	1025	Error on rename of 't2_new' (errno: -1 "Internal error < 0 (Not system error)")
+drop table t1;
+DROP DATABASE test;
+CREATE DATABASE test;
+USE test;
+SET @@session.storage_engine=MYISAM;
+CREATE TABLE t1(id INT,purchased DATE)PARTITION BY RANGE(YEAR(purchased)) SUBPARTITION BY HASH(TO_DAYS(purchased)) SUBPARTITIONS 2 (PARTITION p0 VALUES LESS THAN MAXVALUE (SUBPARTITION sp0 DATA DIRECTORY='/tmp/not-existing' INDEX DIRECTORY='/tmp/not-existing',SUBPARTITION sp1));
+ERROR HY000: Can't create/write to file '/tmp/not-existing/t1#P#p0#SP#sp0.MYI' (Errcode: 2 "No such file or directory")
+CREATE TABLE t1(id INT,purchased DATE)PARTITION BY RANGE(YEAR(purchased)) SUBPARTITION BY HASH(TO_DAYS(purchased)) SUBPARTITIONS 2 (PARTITION p0 VALUES LESS THAN MAXVALUE (SUBPARTITION sp0,SUBPARTITION sp1));
+ALTER TABLE t1 ENGINE=InnoDB;
+drop table t1;

mysql-test/suite/innodb/t/innodb-mdev7046.test

+--source include/have_innodb.inc
+--source include/have_partition.inc
+
+# Ignore OS errors
+call mtr.add_suppression("InnoDB: File ./test/t1*");
+call mtr.add_suppression("nnoDB: Error number*");
+
+# MDEV-7046: MySQL#74480 - Failing assertion: os_file_status(newpath, &exists, &type) 
+# after Operating system error number 36 in a file operation
+
+USE test;
+create table t1(f1 INT,f2 INT,f3 CHAR (10),primary key(f1,f2)) partition by range(f1) subpartition by hash(f2) subpartitions 2 (partition p1 values less than (0),partition p2 values less than (2),partition p3 values less than (2147483647));
+--replace_regex /'.*t2_new.*'/'t2_new'/
+--error 7
+RENAME TABLE t1 TO `t2_new..............................................end`;
+alter table t1 engine=innodb;
+--replace_regex /'.*t2_new.*'/'t2_new'/
+--error 1025
+RENAME TABLE t1 TO `t2_new..............................................end`;
+--replace_regex /'.*t2_new.*'/'t2_new'/
+show warnings;
+drop table t1;
+
+DROP DATABASE test;CREATE DATABASE test;USE test;
+SET @@session.storage_engine=MYISAM;
+--error 1
+CREATE TABLE t1(id INT,purchased DATE)PARTITION BY RANGE(YEAR(purchased)) SUBPARTITION BY HASH(TO_DAYS(purchased)) SUBPARTITIONS 2 (PARTITION p0 VALUES LESS THAN MAXVALUE (SUBPARTITION sp0 DATA DIRECTORY='/tmp/not-existing' INDEX DIRECTORY='/tmp/not-existing',SUBPARTITION sp1));
+CREATE TABLE t1(id INT,purchased DATE)PARTITION BY RANGE(YEAR(purchased)) SUBPARTITION BY HASH(TO_DAYS(purchased)) SUBPARTITIONS 2 (PARTITION p0 VALUES LESS THAN MAXVALUE (SUBPARTITION sp0,SUBPARTITION sp1));
+ALTER TABLE t1 ENGINE=InnoDB;
+
+drop table t1;

storage/innobase/include/os0file.h

@@ -139,6 +139,7 @@ enum os_file_create_t {
 /* @} */
 
 /** Error codes from os_file_get_last_error @{ */
+#define	OS_FILE_NAME_TOO_LONG		36
 #define	OS_FILE_NOT_FOUND		71
 #define	OS_FILE_DISK_FULL		72
 #define	OS_FILE_ALREADY_EXISTS		73

storage/innobase/os/os0file.cc

@@ -467,6 +467,8 @@ os_file_get_last_error_low(
 		return(OS_FILE_OPERATION_ABORTED);
 	} else if (err == ERROR_ACCESS_DENIED) {
 		return(OS_FILE_ACCESS_VIOLATION);
+	} else if (err == ERROR_BUFFER_OVERFLOW) {
+		return(OS_FILE_NAME_TOO_LONG);
 	} else {
 		return(OS_FILE_ERROR_MAX + err);
 	}
@@ -528,6 +530,8 @@ os_file_get_last_error_low(
 		return(OS_FILE_NOT_FOUND);
 	case EEXIST:
 		return(OS_FILE_ALREADY_EXISTS);
+	case ENAMETOOLONG:
+		return(OS_FILE_NAME_TOO_LONG);
 	case EXDEV:
 	case ENOTDIR:
 	case EISDIR:
@@ -3049,7 +3053,7 @@ os_file_status(
 	struct _stat64	statinfo;
 
 	ret = _stat64(path, &statinfo);
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 		*exists = FALSE;
 		return(TRUE);
@@ -3077,7 +3081,7 @@ os_file_status(
 	struct stat	statinfo;
 
 	ret = stat(path, &statinfo);
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 		*exists = FALSE;
 		return(TRUE);

storage/xtradb/include/os0file.h

@@ -144,6 +144,7 @@ enum os_file_create_t {
 /* @} */
 
 /** Error codes from os_file_get_last_error @{ */
+#define	OS_FILE_NAME_TOO_LONG		36
 #define	OS_FILE_NOT_FOUND		71
 #define	OS_FILE_DISK_FULL		72
 #define	OS_FILE_ALREADY_EXISTS		73

storage/xtradb/os/os0file.cc

@@ -565,6 +565,8 @@ os_file_get_last_error_low(
 		return(OS_FILE_OPERATION_ABORTED);
 	} else if (err == ERROR_ACCESS_DENIED) {
 		return(OS_FILE_ACCESS_VIOLATION);
+	} else if (err == ERROR_BUFFER_OVERFLOW) {
+		return(OS_FILE_NAME_TOO_LONG);
 	} else {
 		return(OS_FILE_ERROR_MAX + err);
 	}
@@ -626,6 +628,8 @@ os_file_get_last_error_low(
 		return(OS_FILE_NOT_FOUND);
 	case EEXIST:
 		return(OS_FILE_ALREADY_EXISTS);
+	case ENAMETOOLONG:
+		return(OS_FILE_NAME_TOO_LONG);
 	case EXDEV:
 	case ENOTDIR:
 	case EISDIR:
@@ -3219,7 +3223,7 @@ os_file_status(
 	struct _stat64	statinfo;
 
 	ret = _stat64(path, &statinfo);
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 		*exists = FALSE;
 		return(TRUE);
@@ -3247,7 +3251,7 @@ os_file_status(
 	struct stat	statinfo;
 
 	ret = stat(path, &statinfo);
-	if (ret && (errno == ENOENT || errno == ENOTDIR)) {
+	if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) {
 		/* file does not exist */
 		*exists = FALSE;
 		return(TRUE);