[PATCH] epoll bits 0.34

 - Some constant adjusted
 - Comments added
 - Better hash initialization
 - Correct timeout setup
 - Added __KERNEL__ bypass to avoid userspace inclusion problems
 - Cleaned up locking
 - Function poll_init_wait() now calls poll_init_wait_ex()
 - Event return fix ( Jay Vosburgh )
 - Use <linux/hash.h> for the hash

fs/eventpoll.c

@@ -25,6 +25,7 @@
 #include <linux/smp_lock.h>
 #include <linux/string.h>
 #include <linux/list.h>
+#include <linux/hash.h>
 #include <linux/spinlock.h>
 #include <linux/wait.h>
 #include <asm/bitops.h>
@@ -58,11 +59,12 @@
 #endif /* #if DEBUG_DPI != 0 */
 
 
-/* Maximum storage for the eventpoll interest set */
-#define EP_MAX_FDS_SIZE (1024 * 128)
+/* Maximum size of the hash in bits ( 2^N ) */
+#define EP_MAX_HASH_BITS 17
+
+/* Minimum size of the hash in bits ( 2^N ) */
+#define EP_MIN_HASH_BITS 9
 
-/* We don't want the hash to be smaller than this */
-#define EP_MIN_HASH_SIZE 101
 /*
  * Event buffer dimension used to cache events before sending them in
  * userspace with a __copy_to_user(). The event buffer is in stack,
@@ -77,7 +79,7 @@
 #define EP_HENTRY_X_PAGE (PAGE_SIZE / sizeof(struct list_head))
 
 /* Maximum size of the hash in pages */
-#define EP_MAX_HPAGES (EP_MAX_FDS_SIZE / EP_HENTRY_X_PAGE + 1)
+#define EP_MAX_HPAGES ((1 << EP_MAX_HASH_BITS) / EP_HENTRY_X_PAGE + 1)
 
 /* Macro to allocate a "struct epitem" from the slab cache */
 #define DPI_MEM_ALLOC()	(struct epitem *) kmem_cache_alloc(dpi_cache, SLAB_KERNEL)
@@ -127,7 +129,7 @@ struct eventpoll {
 	struct list_head rdllist;
 
 	/* Size of the hash */
-	int hsize;
+	unsigned int hashbits;
 
 	/* Number of pages currently allocated for the hash */
 	int nhpages;
@@ -186,22 +188,20 @@ struct epitem {
 
 
 
-
-static int ep_is_prime(int n);
+static unsigned int ep_get_hash_bits(unsigned int hintsize);
 static int ep_getfd(int *efd, struct inode **einode, struct file **efile);
 static int ep_alloc_pages(char **pages, int numpages);
 static int ep_free_pages(char **pages, int numpages);
-static int ep_file_init(struct file *file, int hsize);
-static int ep_hash_index(struct eventpoll *ep, struct file *file);
-static struct list_head *ep_hash_entry(struct eventpoll *ep, int index);
-static int ep_init(struct eventpoll *ep, int hsize);
+static int ep_file_init(struct file *file, unsigned int hashbits);
+static unsigned int ep_hash_index(struct eventpoll *ep, struct file *file);
+static struct list_head *ep_hash_entry(struct eventpoll *ep, unsigned int index);
+static int ep_init(struct eventpoll *ep, unsigned int hashbits);
 static void ep_free(struct eventpoll *ep);
 static struct epitem *ep_find(struct eventpoll *ep, struct file *file);
 static void ep_use_epitem(struct epitem *dpi);
 static void ep_release_epitem(struct epitem *dpi);
 static void ep_ptable_queue_proc(void *priv, wait_queue_head_t *whead);
 static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfile);
-static unsigned int ep_get_file_events(struct file *file);
 static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int events);
 static int ep_unlink(struct eventpoll *ep, struct epitem *dpi);
 static int ep_remove(struct eventpoll *ep, struct epitem *dpi);
@@ -217,6 +217,7 @@ static struct super_block *eventpollfs_get_sb(struct file_system_type *fs_type,
 					      int flags, char *dev_name, void *data);
 
 
+
 /* Use to link togheter all the "struct eventpoll" */
 static struct list_head eplist;
 
@@ -252,21 +253,17 @@ static struct dentry_operations eventpollfs_dentry_operations = {
 
 
 
-/* Report if the number is prime. Needed to correctly size the hash  */
-static int ep_is_prime(int n)
+/*
+ * Calculate the size of the hash in bits. The returned size will be
+ * bounded between EP_MIN_HASH_BITS and EP_MAX_HASH_BITS.
+ */
+static unsigned int ep_get_hash_bits(unsigned int hintsize)
 {
+	unsigned int i, val;
 
-	if (n > 3) {
-		if (n & 1) {
-			int i, hn = n / 2;
-
-			for (i = 3; i < hn; i += 2)
-				if (!(n % i))
-					return 0;
-		} else
-			return 0;
-	}
-	return 1;
+	for (i = 0, val = 1; val < hintsize && i < 8 * sizeof(int); i++, val <<= 1);
+	return i <  EP_MIN_HASH_BITS ?  EP_MIN_HASH_BITS:
+		(i > EP_MAX_HASH_BITS ? EP_MAX_HASH_BITS: i);
 }
 
 
@@ -298,21 +295,23 @@ void ep_notify_file_close(struct file *file)
 
 /*
  * It opens an eventpoll file descriptor by suggesting a storage of "size"
- * file descriptors. It is the kernel part of the userspace epoll_create(2).
+ * file descriptors. The size parameter is just an hint about how to size
+ * data structures. It won't prevent the user to store more than "size"
+ * file descriptors inside the epoll interface. It is the kernel part of
+ * the userspace epoll_create(2).
  */
 asmlinkage int sys_epoll_create(int size)
 {
 	int error, fd;
+	unsigned int hashbits;
 	struct inode *inode;
 	struct file *file;
 
 	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d)\n",
 		     current, size));
 
-	/* Search the nearest prime number higher than "size" */
-	for (; !ep_is_prime(size); size++);
-	if (size < EP_MIN_HASH_SIZE)
-		size = EP_MIN_HASH_SIZE;
+	/* Correctly size the hash */
+	hashbits = ep_get_hash_bits((unsigned int) size);
 
 	/*
 	 * Creates all the items needed to setup an eventpoll file. That is,
@@ -323,7 +322,7 @@ asmlinkage int sys_epoll_create(int size)
 		goto eexit_1;
 
 	/* Setup the file internal data structure ( "struct eventpoll" ) */
-	error = ep_file_init(file, size);
+	error = ep_file_init(file, hashbits);
 	if (error)
 		goto eexit_2;
 
@@ -457,6 +456,10 @@ asmlinkage int sys_epoll_wait(int epfd, struct pollfd *events, int maxevents,
 	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d)\n",
 		     current, epfd, events, maxevents, timeout));
 
+	/* The maximum number of event must be greater than zero */
+	if (maxevents <= 0)
+		return -EINVAL;
+
 	/* Verify that the area passed by the user is writeable */
 	if ((error = verify_area(VERIFY_WRITE, events, maxevents * sizeof(struct pollfd))))
 		goto eexit_1;
@@ -602,7 +605,7 @@ static int ep_free_pages(char **pages, int numpages)
 }
 
 
-static int ep_file_init(struct file *file, int hsize)
+static int ep_file_init(struct file *file, unsigned int hashbits)
 {
 	int error;
 	unsigned long flags;
@@ -613,7 +616,7 @@ static int ep_file_init(struct file *file, int hsize)
 
 	memset(ep, 0, sizeof(*ep));
 
-	error = ep_init(ep, hsize);
+	error = ep_init(ep, hashbits);
 	if (error) {
 		kfree(ep);
 		return error;
@@ -635,17 +638,17 @@ static int ep_file_init(struct file *file, int hsize)
 /*
  * Calculate the index of the hash relative to "file".
  */
-static int ep_hash_index(struct eventpoll *ep, struct file *file)
+static unsigned int ep_hash_index(struct eventpoll *ep, struct file *file)
 {
 
-	return (int) ((((unsigned long) file) / sizeof(struct file)) % ep->hsize);
+	return (unsigned int) hash_ptr(file, ep->hashbits);
 }
 
 
 /*
  * Returns the hash entry ( struct list_head * ) of the passed index.
  */
-static struct list_head *ep_hash_entry(struct eventpoll *ep, int index)
+static struct list_head *ep_hash_entry(struct eventpoll *ep, unsigned int index)
 {
 
 	return (struct list_head *) (ep->hpages[index / EP_HENTRY_X_PAGE] +
@@ -653,9 +656,10 @@ static struct list_head *ep_hash_entry(struct eventpoll *ep, int index)
 }
 
 
-static int ep_init(struct eventpoll *ep, int hsize)
+static int ep_init(struct eventpoll *ep, unsigned int hashbits)
 {
-	int error, i;
+	int error;
+	unsigned int i, hsize;
 
 	INIT_LIST_HEAD(&ep->llink);
 	rwlock_init(&ep->lock);
@@ -664,14 +668,15 @@ static int ep_init(struct eventpoll *ep, int hsize)
 	INIT_LIST_HEAD(&ep->rdllist);
 
 	/* Hash allocation and setup */
-	ep->hsize = hsize;
-	ep->nhpages = hsize / EP_HENTRY_X_PAGE + (hsize % EP_HENTRY_X_PAGE ? 1: 0);
+	hsize = 1 << hashbits;
+	ep->hashbits = hashbits;
+	ep->nhpages = (int) (hsize / EP_HENTRY_X_PAGE + (hsize % EP_HENTRY_X_PAGE ? 1: 0));
 	error = ep_alloc_pages(ep->hpages, ep->nhpages);
 	if (error)
 		goto eexit_1;
 
 	/* Initialize hash buckets */
-	for (i = 0; i < ep->hsize; i++)
+	for (i = 0; i < hsize; i++)
 		INIT_LIST_HEAD(ep_hash_entry(ep, i));
 
 	return 0;
@@ -682,7 +687,7 @@ static int ep_init(struct eventpoll *ep, int hsize)
 
 static void ep_free(struct eventpoll *ep)
 {
-	int i;
+	unsigned int i, hsize;
 	unsigned long flags;
 	struct list_head *lsthead;
 
@@ -690,7 +695,7 @@ static void ep_free(struct eventpoll *ep)
 	 * Walks through the whole hash by unregistering file callbacks and
 	 * freeing each "struct epitem".
 	 */
-	for (i = 0; i < ep->hsize; i++) {
+	for (i = 0, hsize = 1 << ep->hashbits; i < hsize; i++) {
 		lsthead = ep_hash_entry(ep, i);
 
 		/*
@@ -723,7 +728,6 @@ static void ep_free(struct eventpoll *ep)
 	write_unlock_irqrestore(&eplock, flags);
 
 	/* Free hash pages */
-	if (ep->nhpages > 0)
 	ep_free_pages(ep->hpages, ep->nhpages);
 }
 
@@ -827,22 +831,26 @@ static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfil
 		dpi->wait[i].base = dpi;
 	}
 
-	/* Attach the item to the poll hooks */
+	/* Initialize the poll table using the queue callback */
 	poll_initwait_ex(&pt, 1, ep_ptable_queue_proc, dpi);
-	revents = tfile->f_op->poll(tfile, &pt);
-	poll_freewait(&pt);
 
 	/* We have to drop the new item inside our item list to keep track of it */
 	write_lock_irqsave(&ep->lock, flags);
 
+	/* Add the current item to the hash table */
 	list_add(&dpi->llink, ep_hash_entry(ep, ep_hash_index(ep, tfile)));
 
+	/* Attach the item to the poll hooks and get current event bits */
+	revents = tfile->f_op->poll(tfile, &pt);
+
 	/* If the file is already "ready" we drop it inside the ready list */
 	if ((revents & pfd->events) && !EP_IS_LINKED(&dpi->rdllink))
 		list_add(&dpi->rdllink, &ep->rdllist);
 
 	write_unlock_irqrestore(&ep->lock, flags);
 
+	poll_freewait(&pt);
+
 	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_insert(%p, %d)\n",
 		     current, ep, pfd->fd));
 
@@ -854,12 +862,13 @@ static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfil
 
 
 /*
- * Returns the current events of the given file. It uses the special
- * poll table initialization to avoid any poll queue insertion.
+ * Modify the interest event mask by dropping an event if the new mask
+ * has a match in the current file status.
  */
-static unsigned int ep_get_file_events(struct file *file)
+static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int events)
 {
 	unsigned int revents;
+	unsigned long flags;
 	poll_table pt;
 
 	/*
@@ -870,26 +879,12 @@ static unsigned int ep_get_file_events(struct file *file)
 	 */
 	poll_initwait_ex(&pt, 0, NULL, NULL);
 
-	revents = file->f_op->poll(file, &pt);
-
-	poll_freewait(&pt);
-	return revents;
-}
-
-
-/*
- * Modify the interest event mask by dropping an event if the new mask
- * has a match in the current file status.
- */
-static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int events)
-{
-	unsigned int revents;
-	unsigned long flags;
-
-	revents = ep_get_file_events(dpi->file);
-
 	write_lock_irqsave(&ep->lock, flags);
 
+	/* Get current event bits */
+	revents = dpi->file->f_op->poll(dpi->file, &pt);
+
+	/* Set the new event interest mask */
 	dpi->pfd.events = events;
 
 	/* If the file is already "ready" we drop it inside the ready list */
@@ -898,6 +893,8 @@ static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int even
 
 	write_unlock_irqrestore(&ep->lock, flags);
 
+	poll_freewait(&pt);
+
 	return 0;
 }
 
@@ -1068,12 +1065,20 @@ static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int m
 
 	write_lock_irqsave(&ep->lock, flags);
 
-	for (eventcnt = 0, ebufcnt = 0; eventcnt < maxevents && !list_empty(lsthead);) {
+	for (eventcnt = 0, ebufcnt = 0; (eventcnt + ebufcnt) < maxevents && !list_empty(lsthead);) {
 		struct epitem *dpi = list_entry(lsthead->next, struct epitem, rdllink);
 
 		/* Remove the item from the ready list */
 		EP_LIST_DEL(&dpi->rdllink);
 
+		/*
+		 * If the item is not linked to the main has table this means that
+		 * it's on the way to be removed and we don't want to send events
+		 * to such file descriptor.
+		 */
+		if (!EP_IS_LINKED(&dpi->llink))
+			continue;
+
 		/* Fetch event bits from the signaled file */
 		revents = dpi->file->f_op->poll(dpi->file, &pt);
 
@@ -1126,12 +1131,10 @@ static int ep_poll(struct eventpoll *ep, struct pollfd *events, int maxevents,
 	wait_queue_t wait;
 
 	/*
-	 * Calculate the timeout by checking for the "infinite" value ( -1 )
-	 * and the overflow condition ( > MAX_SCHEDULE_TIMEOUT / HZ ). The
-	 * passed timeout is in milliseconds, that why (t * HZ) / 1000.
+	 * Calculate the timeout by checking for the "infinite" value ( -1 ).
+	 * The passed timeout is in milliseconds, that why (t * HZ) / 1000.
 	 */
-	jtimeout = timeout == -1 || timeout > MAX_SCHEDULE_TIMEOUT / HZ ?
-		MAX_SCHEDULE_TIMEOUT: (timeout * HZ) / 1000;
+	jtimeout = timeout == -1 ? MAX_SCHEDULE_TIMEOUT: (timeout * HZ) / 1000;
 
 retry:
 	write_lock_irqsave(&ep->lock, flags);

include/linux/eventpoll.h

@@ -14,10 +14,6 @@
 #ifndef _LINUX_EVENTPOLL_H
 #define _LINUX_EVENTPOLL_H
 
-/* Forward declarations to avoid compiler errors */
-struct file;
-struct pollfd;
-
 
 /* Valid opcodes to issue to sys_epoll_ctl() */
 #define EP_CTL_ADD 1
@@ -25,6 +21,13 @@ struct pollfd;
 #define EP_CTL_MOD 3
 
 
+#ifdef __KERNEL__
+
+/* Forward declarations to avoid compiler errors */
+struct file;
+struct pollfd;
+
+
 /* Kernel space functions implementing the user space "epoll" API */
 asmlinkage int sys_epoll_create(int size);
 asmlinkage int sys_epoll_ctl(int epfd, int op, int fd, unsigned int events);
@@ -34,6 +37,7 @@ asmlinkage int sys_epoll_wait(int epfd, struct pollfd *events, int maxevents,
 /* Used in fs/file_table.c:__fput() to unlink files from the eventpoll interface */
 void ep_notify_file_close(struct file *file);
 
+#endif /* #ifdef __KERNEL__ */
 
-#endif
+#endif /* #ifndef _LINUX_EVENTPOLL_H */
 

include/linux/poll.h

@@ -28,15 +28,6 @@ static inline void poll_wait(struct file * filp, wait_queue_head_t * wait_addres
 		__pollwait(filp, wait_address, p);
 }
 
-static inline void poll_initwait(poll_table* pt)
-{
-	pt->queue = 1;
-	pt->qproc = NULL;
-	pt->priv = NULL;
-	pt->error = 0;
-	pt->table = NULL;
-}
-
 static inline void poll_initwait_ex(poll_table* pt, int queue,
 				    void (*qproc)(void *, wait_queue_head_t *),
 				    void *priv)
@@ -48,6 +39,12 @@ static inline void poll_initwait_ex(poll_table* pt, int queue,
 	pt->table = NULL;
 }
 
+static inline void poll_initwait(poll_table* pt)
+{
+
+	poll_initwait_ex(pt, 1, NULL, NULL);
+}
+
 extern void poll_freewait(poll_table* pt);