NTFS: 2.0.20 - Support non-resident directory index bitmaps, fix page leak in readdir.

- Move the directory index bitmap to use an attribute inode instead of
  having special fields for it inside the ntfs inode structure. This
  means that the index bitmaps now use the page cache for i/o, too,
  and also as a side effect we get support for non-resident index
  bitmaps for free.
- Simplify/cleanup error handling in fs/ntfs/dir.c::ntfs_readdir() and
  fix a page leak that manifested itself in some cases.
- Add fs/ntfs/inode.c::ntfs_put_inode(), which we need to release the
  index bitmap inode on the final iput().

Documentation/filesystems/ntfs.txt

@@ -247,6 +247,12 @@ ChangeLog
 
 Note, a technical ChangeLog aimed at kernel hackers is in fs/ntfs/ChangeLog.
 
+2.0.20:
+	- Support non-resident directory index bitmaps. This means we now cope
+	  with huge directories without problems.
+	- Fix a page leak that manifested itself in some cases when reading
+	  directory contents.
+	- Internal cleanups.
 2.0.19:
 	- Fix race condition and improvements in block i/o interface.
 	- Optimization when reading compressed files.

fs/ntfs/ChangeLog

@@ -7,10 +7,20 @@ ToDo:
 	  truncate the visible i_size? Will the user just get -E2BIG (or
 	  whatever) on open()? Or will (s)he be able to open() but lseek() and
 	  read() will fail when s_maxbytes is reached? -> Investigate this.
-	- Implement/allow non-resident index bitmaps in dir.c::ntfs_readdir()
-	  and then also consider initialized_size w.r.t. the bitmaps, etc.
 	- Enable NFS exporting of NTFS.
 
+2.0.20 - Support non-resident directory index bitmaps, fix page leak in readdir.
+
+	- Move the directory index bitmap to use an attribute inode instead of
+	  having special fields for it inside the ntfs inode structure. This
+	  means that the index bitmaps now use the page cache for i/o, too,
+	  and also as a side effect we get support for non-resident index
+	  bitmaps for free.
+	- Simplify/cleanup error handling in fs/ntfs/dir.c::ntfs_readdir() and
+	  fix a page leak that manifested itself in some cases.
+	- Add fs/ntfs/inode.c::ntfs_put_inode(), which we need to release the
+	  index bitmap inode on the final iput().
+
 2.0.19 - Fix race condition, improvements, and optimizations in i/o interface.
 
 	- Apply block optimization added to fs/ntfs/aops.c::ntfs_read_block()

fs/ntfs/Makefile

@@ -5,7 +5,7 @@ obj-$(CONFIG_NTFS_FS) += ntfs.o
 ntfs-objs := aops.o attrib.o compress.o debug.o dir.o file.o inode.o mft.o \
 	     mst.o namei.o super.o sysctl.o time.o unistr.o upcase.o
 
-EXTRA_CFLAGS = -DNTFS_VERSION=\"2.0.19\"
+EXTRA_CFLAGS = -DNTFS_VERSION=\"2.0.20\"
 
 ifeq ($(CONFIG_NTFS_DEBUG),y)
 EXTRA_CFLAGS += -DDEBUG

fs/ntfs/inode.c

@@ -329,12 +329,9 @@ static void __ntfs_init_inode(struct super_block *sb, ntfs_inode *ni)
 	ni->attr_list_size = 0;
 	ni->attr_list = NULL;
 	init_run_list(&ni->attr_list_rl);
+	ni->_IDM(bmp_ino) = NULL;
 	ni->_IDM(index_block_size) = 0;
 	ni->_IDM(index_vcn_size) = 0;
-	ni->_IDM(bmp_size) = 0;
-	ni->_IDM(bmp_initialized_size) = 0;
-	ni->_IDM(bmp_allocated_size) = 0;
-	init_run_list(&ni->_IDM(bmp_rl));
 	ni->_IDM(index_block_size_bits) = 0;
 	ni->_IDM(index_vcn_size_bits) = 0;
 	init_MUTEX(&ni->extent_lock);
@@ -680,6 +677,8 @@ static int ntfs_read_locked_inode(struct inode *vi)
 	 * in ntfs_ino->attr_list and it is ntfs_ino->attr_list_size bytes.
 	 */
 	if (S_ISDIR(vi->i_mode)) {
+		struct inode *bvi;
+		ntfs_inode *bni;
 		INDEX_ROOT *ir;
 		char *ir_end, *index_end;
 
@@ -787,7 +786,8 @@ static int ntfs_read_locked_inode(struct inode *vi)
 
 		if (!(ir->index.flags & LARGE_INDEX)) {
 			/* No index allocation. */
-			vi->i_size = ni->initialized_size = 0;
+			vi->i_size = ni->initialized_size =
+					ni->allocated_size = 0;
 			goto skip_large_dir_stuff;
 		} /* LARGE_INDEX: Index allocation present. Setup state. */
 		NInoSetIndexAllocPresent(ni);
@@ -832,63 +832,31 @@ static int ntfs_read_locked_inode(struct inode *vi)
 				ctx->attr->_ANR(initialized_size));
 		ni->allocated_size = sle64_to_cpu(
 				ctx->attr->_ANR(allocated_size));
-		/* Find bitmap attribute. */
-		reinit_attr_search_ctx(ctx);
-		if (!lookup_attr(AT_BITMAP, I30, 4, CASE_SENSITIVE, 0, NULL, 0,
-				ctx)) {
-			ntfs_error(vi->i_sb, "$BITMAP attribute is not "
-					"present but it must be.");
+
+		/* Get the index bitmap attribute inode. */
+		bvi = ntfs_attr_iget(vi, AT_BITMAP, I30, 4);
+		if (unlikely(IS_ERR(bvi))) {
+			ntfs_error(vi->i_sb, "Failed to get bitmap attribute.");
+			err = PTR_ERR(bvi);
 			goto unm_err_out;
 		}
-		if (ctx->attr->flags & (ATTR_COMPRESSION_MASK |
-				ATTR_IS_ENCRYPTED | ATTR_IS_SPARSE)) {
+		ni->_IDM(bmp_ino) = bvi;
+		bni = NTFS_I(bvi);
+		if (NInoCompressed(bni) || NInoEncrypted(bni) ||
+				NInoSparse(bni)) {
 			ntfs_error(vi->i_sb, "$BITMAP attribute is compressed "
 					"and/or encrypted and/or sparse.");
 			goto unm_err_out;
 		}
-		if (ctx->attr->non_resident) {
-			NInoSetBmpNonResident(ni);
-			if (ctx->attr->_ANR(lowest_vcn)) {
-				ntfs_error(vi->i_sb, "First extent of $BITMAP "
-						"attribute has non zero "
-						"lowest_vcn. Inode is corrupt. "
-						"You should run chkdsk.");
-				goto unm_err_out;
-			}
-			ni->_IDM(bmp_size) = sle64_to_cpu(
-					ctx->attr->_ANR(data_size));
-			ni->_IDM(bmp_initialized_size) = sle64_to_cpu(
-					ctx->attr->_ANR(initialized_size));
-			ni->_IDM(bmp_allocated_size) = sle64_to_cpu(
-					ctx->attr->_ANR(allocated_size));
-			/*
-			 * Setup the run list. No need for locking as we have
-			 * exclusive access to the inode at this time.
-			 */
-			ni->_IDM(bmp_rl).rl = decompress_mapping_pairs(vol,
-					ctx->attr, NULL);
-			if (IS_ERR(ni->_IDM(bmp_rl).rl)) {
-				err = PTR_ERR(ni->_IDM(bmp_rl).rl);
-				ni->_IDM(bmp_rl).rl = NULL;
-				ntfs_error(vi->i_sb, "Mapping pairs "
-						"decompression failed with "
-						"error code %i.", -err);
-				goto unm_err_out;
-			}
-		} else
-			ni->_IDM(bmp_size) = ni->_IDM(bmp_initialized_size) =
-					ni->_IDM(bmp_allocated_size) =
-					le32_to_cpu(
-					ctx->attr->_ARA(value_length));
 		/* Consistency check bitmap size vs. index allocation size. */
-		if (ni->_IDM(bmp_size) << 3 < vi->i_size >>
-				ni->_IDM(index_block_size_bits)) {
-			ntfs_error(vi->i_sb, "$I30 bitmap too small (0x%Lx) "
+		if ((bvi->i_size << 3) < (vi->i_size >>
+				ni->_IDM(index_block_size_bits))) {
+			ntfs_error(vi->i_sb, "Index bitmap too small (0x%Lx) "
 					"for index allocation (0x%Lx).",
-					(long long)ni->_IDM(bmp_size) << 3,
-					vi->i_size);
+					bvi->i_size << 3, vi->i_size);
 			goto unm_err_out;
 		}
+
 skip_large_dir_stuff:
 		/* Everyone gets read and scan permissions. */
 		vi->i_mode |= S_IRUGO | S_IXUGO;
@@ -1271,7 +1239,7 @@ static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi)
 						"will probably cause problems "
 						"when trying to access the "
 						"file. Please notify "
-						"linux-ntfs-dev@ lists.sf.net "
+						"linux-ntfs-dev@lists.sf.net "
 						"that you saw this message.");
 		}
 	}
@@ -1785,6 +1753,32 @@ int ntfs_commit_inode(ntfs_inode *ni)
 	return 0;
 }
 
+/**
+ * ntfs_put_inode - handler for when the inode reference count is decremented
+ * @vi:		vfs inode
+ *
+ * The VFS calls ntfs_put_inode() every time the inode reference count (i_count)
+ * is about to be decremented (but before the decrement itself.
+ *
+ * If the inode @vi is a directory with a single reference, we need to put the
+ * attribute inode for the directory index bitmap, if it is present, otherwise
+ * the directory inode would remain pinned for ever (or rather until umount()
+ * time.
+ */
+void ntfs_put_inode(struct inode *vi)
+{
+	if (S_ISDIR(vi->i_mode) && (atomic_read(&vi->i_count) == 2)) {
+		ntfs_inode *ni;
+
+		ni = NTFS_I(vi);
+		if (NInoIndexAllocPresent(ni) && ni->_IDM(bmp_ino)) {
+			iput(ni->_IDM(bmp_ino));
+			ni->_IDM(bmp_ino) = NULL;
+		}
+	}
+	return;
+}
+
 void __ntfs_clear_inode(ntfs_inode *ni)
 {
 	int err;
@@ -1866,12 +1860,7 @@ void ntfs_clear_big_inode(struct inode *vi)
 
 	__ntfs_clear_inode(ni);
 
-	if (S_ISDIR(vi->i_mode)) {
-		down_write(&ni->_IDM(bmp_rl).lock);
-		if (ni->_IDM(bmp_rl).rl)
-			ntfs_free(ni->_IDM(bmp_rl).rl);
-		up_write(&ni->_IDM(bmp_rl).lock);
-	} else if (NInoAttr(ni)) {
+	if (NInoAttr(ni)) {
 		/* Release the base inode if we are holding it. */
 		if (ni->nr_extents == -1) {
 			iput(VFS_I(ni->_INE(base_ntfs_ino)));

fs/ntfs/inode.h

@@ -92,14 +92,11 @@ struct _ntfs_inode {
 	run_list attr_list_rl;	/* Run list for the attribute list value. */
 	union {
 		struct { /* It is a directory or $MFT. */
+			struct inode *bmp_ino;	/* Attribute inode for the
+						   directory index $BITMAP. */
 			u32 index_block_size;	/* Size of an index block. */
 			u32 index_vcn_size;	/* Size of a vcn in this
 						   directory index. */
-			s64 bmp_size;		/* Size of the $I30 bitmap. */
-			s64 bmp_initialized_size; /* Copy from $I30 bitmap. */
-			s64 bmp_allocated_size;	/* Copy from $I30 bitmap. */
-			run_list bmp_rl;	/* Run list for the $I30 bitmap
-						   if it is non-resident. */
 			u8 index_block_size_bits; /* Log2 of the above. */
 			u8 index_vcn_size_bits;	/* Log2 of the above. */
 		} SN(idm);
@@ -165,7 +162,6 @@ typedef enum {
 	NI_Sparse,		/* 1: Unnamed data attr is sparse (f).
 				   1: Create sparse files by default (d).
 				   1: Attribute is sparse (a). */
-	NI_BmpNonResident,	/* 1: $I30 bitmap attr is non resident (d). */
 } ntfs_inode_state_bits;
 
 /*
@@ -203,7 +199,6 @@ NINO_FNS(IndexAllocPresent)
 NINO_FNS(Compressed)
 NINO_FNS(Encrypted)
 NINO_FNS(Sparse)
-NINO_FNS(BmpNonResident)
 
 /*
  * The full structure containing a ntfs_inode and a vfs struct inode. Used for
@@ -247,6 +242,8 @@ extern void ntfs_read_inode_mount(struct inode *vi);
 
 extern void ntfs_dirty_inode(struct inode *vi);
 
+extern void ntfs_put_inode(struct inode *vi);
+
 extern int ntfs_show_options(struct seq_file *sf, struct vfsmount *mnt);
 
 #endif /* _LINUX_NTFS_FS_INODE_H */

fs/ntfs/super.c

@@ -1263,10 +1263,8 @@ struct super_operations ntfs_sops = {
 	dirty_inode:	ntfs_dirty_inode,	/* VFS: Called from
 						   __mark_inode_dirty(). */
 	//write_inode:	NULL,		/* VFS: Write dirty inode to disk. */
-	//put_inode:	NULL,		/* VFS: Called whenever the reference
-	//				   count (i_count) of the inode is
-	//				   going to be decreased but before the
-	//				   actual decrease. */
+	put_inode:	ntfs_put_inode,	/* VFS: Called just before the inode
+					   reference count is decreased. */
 	//delete_inode:	NULL,		/* VFS: Delete inode from disk. Called
 	//				   when i_count becomes 0 and i_nlink is
 	//				   also 0. */