[PATCH] epoll bits 0.46 ...

- A more uniform poll queueing interface with tips from Manfred

- The f_op->poll() is done outside the irqlock to maintain compatibility
	with existing drivers that assume to be called with irq enabled

- Moved event mask setting inside ep_modify() with tips from John

- Fixed locking to fit the new "poll() outside the lock" approach

- Bufferd userspace event delivery to reduce irq_lock/irq_unlock switching
	rate and to reduce the number of __copy_to_user()

- Comments added

fs/eventpoll.c

@@ -27,6 +27,7 @@
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/spinlock.h>
+#include <linux/rwsem.h>
 #include <linux/wait.h>
 #include <asm/bitops.h>
 #include <asm/uaccess.h>
@@ -65,13 +66,6 @@
 /* Minimum size of the hash in bits ( 2^N ) */
 #define EP_MIN_HASH_BITS 9
 
-/*
- * Event buffer dimension used to cache events before sending them in
- * userspace with a __copy_to_user(). The event buffer is in stack,
- * so keep this size fairly small.
- */
-#define EP_EVENT_BUFF_SIZE 32
-
 /* Maximum number of wait queue we can attach to */
 #define EP_MAX_POLL_QUEUE 2
 
@@ -110,6 +104,17 @@
 /* Get the "struct epitem" from an epoll queue wrapper */
 #define EP_ITEM_FROM_EPQUEUE(p) (container_of(p, struct ep_pqueue, pt)->dpi)
 
+/*
+ * This is used to optimize the event transfer to userspace. Since this
+ * is kept on stack, it should be pretty small.
+ */
+#define EP_MAX_BUF_EVENTS 32
+
+/*
+ * Used to optimize ready items collection by reducing the irqlock/irqunlock
+ * switching rate. This is kept in stack too, so do not go wild with this number.
+ */
+#define EP_MAX_COLLECT_ITEMS 64
 
 
 
@@ -212,11 +217,15 @@ static void ep_release_epitem(struct epitem *dpi);
 static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead, poll_table *pt);
 static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfile);
 static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int events);
+static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *dpi);
 static int ep_unlink(struct eventpoll *ep, struct epitem *dpi);
 static int ep_remove(struct eventpoll *ep, struct epitem *dpi);
 static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync);
 static int ep_eventpoll_close(struct inode *inode, struct file *file);
 static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait);
+static int ep_collect_ready_items(struct eventpoll *ep, struct epitem **adpi, int maxdpi);
+static int ep_send_events(struct eventpoll *ep, struct epitem **adpi, int ndpi,
+			  struct pollfd *events);
 static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int maxevents);
 static int ep_poll(struct eventpoll *ep, struct pollfd *events, int maxevents,
 		   int timeout);
@@ -227,11 +236,21 @@ static struct super_block *eventpollfs_get_sb(struct file_system_type *fs_type,
 
 
 
-/* Use to link togheter all the "struct eventpoll" */
+/*
+ * Use to link togheter all the "struct eventpoll". We need to link
+ * all the available eventpoll structures to be able to perform proper
+ * cleanup in case a file that is stored inside epoll is closed without
+ * previously being removed.
+ */
 static struct list_head eplist;
 
-/* Serialize the access to "eplist" */
-static rwlock_t eplock;
+/*
+ * Serialize the access to "eplist" and also to ep_notify_file_close().
+ * It is read-held when we want to be sure that a given file will not
+ * vanish while we're doing f_op->poll(). When "ep->lock" is taken,
+ * it will nest inside this semaphore.
+ */
+struct rw_semaphore epsem;
 
 /* Slab cache used to allocate "struct epitem" */
 static kmem_cache_t *dpi_cache;
@@ -283,21 +302,24 @@ static unsigned int ep_get_hash_bits(unsigned int hintsize)
  */
 void ep_notify_file_close(struct file *file)
 {
-	unsigned long flags;
 	struct list_head *lnk;
 	struct eventpoll *ep;
 	struct epitem *dpi;
 
-	read_lock_irqsave(&eplock, flags);
+	down_write(&epsem);
 	list_for_each(lnk, &eplist) {
 		ep = list_entry(lnk, struct eventpoll, llink);
 
+		/*
+		 * The ep_find() function increases the "struct epitem" usage count
+		 * so we have to do an ep_remove() + ep_release_epitem().
+		 */
 		while ((dpi = ep_find(ep, file))) {
 			ep_remove(ep, dpi);
 			ep_release_epitem(dpi);
 		}
 	}
-	read_unlock_irqrestore(&eplock, flags);
+	up_write(&epsem);
 }
 
 
@@ -438,14 +460,14 @@ asmlinkage int sys_epoll_ctl(int epfd, int op, int fd, unsigned int events)
 	if (dpi)
 		ep_release_epitem(dpi);
 
-	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %u) = %d\n",
-		     current, epfd, op, fd, events, error));
-
 eexit_3:
 	fput(tfile);
 eexit_2:
 	fput(file);
 eexit_1:
+	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %u) = %d\n",
+		     current, epfd, op, fd, events, error));
+
 	return error;
 }
 
@@ -495,12 +517,12 @@ asmlinkage int sys_epoll_wait(int epfd, struct pollfd *events, int maxevents,
 	/* Time to fish for events ... */
 	error = ep_poll(ep, events, maxevents, timeout);
 
-	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d) = %d\n",
-		     current, epfd, events, maxevents, timeout, error));
-
 eexit_2:
 	fput(file);
 eexit_1:
+	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d) = %d\n",
+		     current, epfd, events, maxevents, timeout, error));
+
 	return error;
 }
 
@@ -616,7 +638,6 @@ static int ep_free_pages(char **pages, int numpages)
 static int ep_file_init(struct file *file, unsigned int hashbits)
 {
 	int error;
-	unsigned long flags;
 	struct eventpoll *ep;
 
 	if (!(ep = kmalloc(sizeof(struct eventpoll), GFP_KERNEL)))
@@ -633,9 +654,9 @@ static int ep_file_init(struct file *file, unsigned int hashbits)
 	file->private_data = ep;
 
 	/* Add the structure to the linked list that links "struct eventpoll" */
-	write_lock_irqsave(&eplock, flags);
+	down_write(&epsem);
 	list_add(&ep->llink, &eplist);
-	write_unlock_irqrestore(&eplock, flags);
+	up_write(&epsem);
 
 	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_file_init() ep=%p\n",
 		     current, ep));
@@ -694,44 +715,47 @@ static int ep_init(struct eventpoll *ep, unsigned int hashbits)
 static void ep_free(struct eventpoll *ep)
 {
 	unsigned int i, hsize;
-	unsigned long flags;
-	struct list_head *lsthead;
+	struct list_head *lsthead, *lnk;
 
 	/*
-	 * Walks through the whole hash by unregistering file callbacks and
-	 * freeing each "struct epitem".
+	 * We need to lock this because we could be hit by
+	 * ep_notify_file_close() while we're freeing the
+	 * "struct eventpoll".
+	 */
+	down_write(&epsem);
+
+	/*
+	 * Walks through the whole hash by unregistering poll callbacks.
 	 */
 	for (i = 0, hsize = 1 << ep->hashbits; i < hsize; i++) {
 		lsthead = ep_hash_entry(ep, i);
 
+		list_for_each(lnk, lsthead) {
+			struct epitem *dpi = list_entry(lnk, struct epitem, llink);
+
+			ep_unregister_pollwait(ep, dpi);
+		}
+	}
+
 	/*
-		 * We need to lock this because we could be hit by
-		 * ep_notify_file_close() while we're freeing this.
+	 * Walks through the whole hash by freeing each "struct epitem". At this
+	 * point we are sure no poll callbacks will be lingering around, so we can
+	 * avoid the lock on "ep->lock".
 	 */
-		write_lock_irqsave(&ep->lock, flags);
+	for (i = 0, hsize = 1 << ep->hashbits; i < hsize; i++) {
+		lsthead = ep_hash_entry(ep, i);
 
 		while (!list_empty(lsthead)) {
 			struct epitem *dpi = list_entry(lsthead->next, struct epitem, llink);
 
-			/* The function ep_unlink() must be called with held lock */
-			ep_unlink(ep, dpi);
-
-			/* We release the lock before releasing the "struct epitem" */
-			write_unlock_irqrestore(&ep->lock, flags);
-
-			ep_release_epitem(dpi);
-
-			/* And then we reaquire the lock ... */
-			write_lock_irqsave(&ep->lock, flags);
+			ep_remove(ep, dpi);
 		}
-
-		write_unlock_irqrestore(&ep->lock, flags);
 	}
 
 	/* Remove the structure to the linked list that links "struct eventpoll" */
-	write_lock_irqsave(&eplock, flags);
 	EP_LIST_DEL(&ep->llink);
-	write_unlock_irqrestore(&eplock, flags);
+
+	up_write(&epsem);
 
 	/* Free hash pages */
 	ep_free_pages(ep->hpages, EP_HASH_PAGES(ep->hashbits));
@@ -839,7 +863,16 @@ static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfil
 
 	/* Initialize the poll table using the queue callback */
 	epq.dpi = dpi;
-	poll_initwait_ex(&epq.pt, ep_ptable_queue_proc);
+	poll_initwait_ex(&epq.pt, ep_ptable_queue_proc, NULL);
+
+	/*
+	 * Attach the item to the poll hooks and get current event bits.
+	 * We can safely use the file* here because its usage count has
+	 * been increased by the caller of this function.
+	 */
+	revents = tfile->f_op->poll(tfile, &epq.pt);
+
+	poll_freewait(&epq.pt);
 
 	/* We have to drop the new item inside our item list to keep track of it */
 	write_lock_irqsave(&ep->lock, flags);
@@ -847,9 +880,6 @@ static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfil
 	/* Add the current item to the hash table */
 	list_add(&dpi->llink, ep_hash_entry(ep, ep_hash_index(ep, tfile)));
 
-	/* Attach the item to the poll hooks and get current event bits */
-	revents = tfile->f_op->poll(tfile, &epq.pt);
-
 	/* If the file is already "ready" we drop it inside the ready list */
 	if ((revents & pfd->events) && !EP_IS_LINKED(&dpi->rdllink)) {
 		list_add_tail(&dpi->rdllink, &ep->rdllist);
@@ -863,8 +893,6 @@ static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfil
 
 	write_unlock_irqrestore(&ep->lock, flags);
 
-	poll_freewait(&epq.pt);
-
 	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_insert(%p, %d)\n",
 		     current, ep, pfd->fd));
 
@@ -884,13 +912,21 @@ static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int even
 	unsigned int revents;
 	unsigned long flags;
 
-	write_lock_irqsave(&ep->lock, flags);
+	/*
+	 * Set the new event interest mask before calling f_op->poll(), otherwise
+	 * a potential race might occur. In fact if we do this operation inside
+	 * the lock, an event might happen between the f_op->poll() call and the
+	 * new event set registering.
+	 */
+	dpi->pfd.events = events;
 
-	/* Get current event bits */
+	/*
+	 * Get current event bits. We can safely use the file* here because
+	 * its usage count has been increased by the caller of this function.
+	 */
 	revents = dpi->file->f_op->poll(dpi->file, NULL);
 
-	/* Set the new event interest mask */
-	dpi->pfd.events = events;
+	write_lock_irqsave(&ep->lock, flags);
 
 	/* If the file is already "ready" we drop it inside the ready list */
 	if ((revents & events) && EP_IS_LINKED(&dpi->llink) &&
@@ -910,20 +946,39 @@ static int ep_modify(struct eventpoll *ep, struct epitem *dpi, unsigned int even
 }
 
 
+/*
+ * This function unregister poll callbacks from the associated file descriptor.
+ * Since this must be called without holding "ep->lock" the atomic exchange trick
+ * will protect us from multiple unregister.
+ */
+static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *dpi)
+{
+	int i, nwait;
+
+	/* This is called without locks, so we need the atomic exchange */
+	nwait = xchg(&dpi->nwait, 0);
+
+	/* Removes poll wait queue hooks */
+	for (i = 0; i < nwait; i++)
+		remove_wait_queue(dpi->wait[i].whead, &dpi->wait[i].wait);
+}
+
+
 /*
  * Unlink the "struct epitem" from all places it might have been hooked up.
  * This function must be called with write IRQ lock on "ep->lock".
  */
 static int ep_unlink(struct eventpoll *ep, struct epitem *dpi)
 {
-	int i;
+	int error;
 
 	/*
 	 * It can happen that this one is called for an item already unlinked.
 	 * The check protect us from doing a double unlink ( crash ).
 	 */
+	error = -ENOENT;
 	if (!EP_IS_LINKED(&dpi->llink))
-		goto not_linked;
+		goto eexit_1;
 
 	/*
 	 * At this point is safe to do the job, unlink the item from our list.
@@ -932,10 +987,6 @@ static int ep_unlink(struct eventpoll *ep, struct epitem *dpi)
 	 */
 	EP_LIST_DEL(&dpi->llink);
 
-	/* Removes poll wait queue hooks */
-	for (i = 0; i < dpi->nwait; i++)
-		remove_wait_queue(dpi->wait[i].whead, &dpi->wait[i].wait);
-
 	/*
 	 * If the item we are going to remove is inside the ready file descriptors
 	 * we want to remove it from this list to avoid stale events.
@@ -943,12 +994,13 @@ static int ep_unlink(struct eventpoll *ep, struct epitem *dpi)
 	if (EP_IS_LINKED(&dpi->rdllink))
 		EP_LIST_DEL(&dpi->rdllink);
 
-not_linked:
+	error = 0;
+eexit_1:
 
-	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_unlink(%p, %d)\n",
-		     current, ep, dpi->pfd.fd));
+	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_unlink(%p, %d) = %d\n",
+		     current, ep, dpi->pfd.fd, error));
 
-	return 0;
+	return error;
 }
 
 
@@ -961,6 +1013,16 @@ static int ep_remove(struct eventpoll *ep, struct epitem *dpi)
 	int error;
 	unsigned long flags;
 
+	/*
+	 * Removes poll wait queue hooks. We _have_ to do this without holding
+	 * the "ep->lock" otherwise a deadlock might occur. This because of the
+	 * sequence of the lock acquisition. Here we do "ep->lock" then the wait
+	 * queue head lock when unregistering the wait queue. The wakeup callback
+	 * will run by holding the wait queue head lock and will call our callback
+	 * that will try to get "ep->lock".
+	 */
+	ep_unregister_pollwait(ep, dpi);
+
 	/* We need to acquire the write IRQ lock before calling ep_unlink() */
 	write_lock_irqsave(&ep->lock, flags);
 
@@ -972,14 +1034,14 @@ static int ep_remove(struct eventpoll *ep, struct epitem *dpi)
 	if (error)
 		goto eexit_1;
 
-	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_remove(%p, %d)\n",
-		     current, ep, dpi->pfd.fd));
-
 	/* At this point it is safe to free the eventpoll item */
 	ep_release_epitem(dpi);
 
 	error = 0;
 eexit_1:
+	DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_remove(%p, %d) = %d\n",
+		     current, ep, dpi->pfd.fd, error));
+
 	return error;
 }
 
@@ -1055,19 +1117,19 @@ static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait)
 
 
 /*
- * Perform the transfer of events to user space. Optimize the copy by
- * caching EP_EVENT_BUFF_SIZE events at a time and then copying it to user space.
+ * Since we have to release the lock during the __copy_to_user() operation and
+ * during the f_op->poll() call, we try to collect the maximum number of items
+ * by reducing the irqlock/irqunlock switching rate.
  */
-static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int maxevents)
+static int ep_collect_ready_items(struct eventpoll *ep, struct epitem **adpi, int maxdpi)
 {
-	int eventcnt, ebufcnt, revents;
+	int ndpi;
 	unsigned long flags;
 	struct list_head *lsthead = &ep->rdllist;
-	struct pollfd eventbuf[EP_EVENT_BUFF_SIZE];
 
 	write_lock_irqsave(&ep->lock, flags);
 
-	for (eventcnt = 0, ebufcnt = 0; (eventcnt + ebufcnt) < maxevents && !list_empty(lsthead);) {
+	for (ndpi = 0; ndpi < maxdpi && !list_empty(lsthead);) {
 		struct epitem *dpi = list_entry(lsthead->next, struct epitem, rdllink);
 
 		/* Remove the item from the ready list */
@@ -1081,41 +1143,112 @@ static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int m
 		if (!EP_IS_LINKED(&dpi->llink))
 			continue;
 
-		/* Fetch event bits from the signaled file */
+		/*
+		 * We need to increase the usage count of the "struct epitem" because
+		 * another thread might call EP_CTL_DEL on this target and make the
+		 * object to vanish underneath our nose.
+		 */
+		ep_use_epitem(dpi);
+
+		adpi[ndpi++] = dpi;
+	}
+
+	write_unlock_irqrestore(&ep->lock, flags);
+
+	return ndpi;
+}
+
+
+/*
+ * This function is called without holding the "ep->lock" since the call to
+ * __copy_to_user() might sleep, and also f_op->poll() might reenable the IRQ
+ * because of the way poll() is traditionally implemented in Linux.
+ */
+static int ep_send_events(struct eventpoll *ep, struct epitem **adpi, int ndpi,
+			  struct pollfd *events)
+{
+	int i, eventcnt, eventbuf, revents;
+	struct epitem *dpi;
+	struct pollfd pfd[EP_MAX_BUF_EVENTS];
+
+	for (i = 0, eventcnt = 0, eventbuf = 0; i < ndpi; i++, adpi++) {
+		dpi = *adpi;
+
+		/* Get the ready file event set */
 		revents = dpi->file->f_op->poll(dpi->file, NULL);
 
 		if (revents & dpi->pfd.events) {
-			eventbuf[ebufcnt] = dpi->pfd;
-			eventbuf[ebufcnt].revents = revents & eventbuf[ebufcnt].events;
-			ebufcnt++;
+			pfd[eventbuf] = dpi->pfd;
+			pfd[eventbuf].revents = revents & pfd[eventbuf].events;
+			eventbuf++;
+			if (eventbuf == EP_MAX_BUF_EVENTS) {
+				if (__copy_to_user(&events[eventcnt], pfd,
+						   eventbuf * sizeof(struct pollfd))) {
+					for (; i < ndpi; i++, adpi++)
+						ep_release_epitem(*adpi);
+					return -EFAULT;
+				}
+				eventcnt += eventbuf;
+				eventbuf = 0;
+			}
+		}
 
-			/* If our buffer page is full we need to flush it to user space */
-			if (ebufcnt == EP_EVENT_BUFF_SIZE) {
-				/*
-				 * We need to drop the irqlock before using the function
-				 * __copy_to_user() because it might fault.
-				 */
-				write_unlock_irqrestore(&ep->lock, flags);
-				if (__copy_to_user(&events[eventcnt], eventbuf,
-						   ebufcnt * sizeof(struct pollfd)))
+		ep_release_epitem(dpi);
+	}
+
+	if (eventbuf) {
+		if (__copy_to_user(&events[eventcnt], pfd,
+				   eventbuf * sizeof(struct pollfd)))
 			return -EFAULT;
-				eventcnt += ebufcnt;
-				ebufcnt = 0;
-				write_lock_irqsave(&ep->lock, flags);
+		eventcnt += eventbuf;
 	}
+
+	return eventcnt;
+}
+
+
+/*
+ * Perform the transfer of events to user space.
+ */
+static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int maxevents)
+{
+	int eventcnt, ndpi, sdpi, maxdpi;
+	struct epitem *adpi[EP_MAX_COLLECT_ITEMS];
+
+	/*
+	 * We need to lock this because we could be hit by
+	 * ep_notify_file_close() while we're transfering
+	 * events to userspace. Read-holding "epsem" will lock
+	 * out  ep_notify_file_close() during the whole
+	 * transfer loop and this will garantie us that the
+	 * file will not vanish underneath our nose when
+	 * we will call f_op->poll() from ep_send_events().
+	 */
+	down_read(&epsem);
+
+	for (eventcnt = 0; eventcnt < maxevents;) {
+		/* Maximum items we can extract this time */
+		maxdpi = min(EP_MAX_COLLECT_ITEMS, maxevents - eventcnt);
+
+		/* Collect/extract ready items */
+		ndpi = ep_collect_ready_items(ep, adpi, maxdpi);
+
+		if (ndpi) {
+			/* Send events to userspace */
+			sdpi = ep_send_events(ep, adpi, ndpi, &events[eventcnt]);
+			if (sdpi < 0) {
+				up_read(&epsem);
+				return sdpi;
 			}
+			eventcnt += sdpi;
 		}
-	write_unlock_irqrestore(&ep->lock, flags);
 
-	/* There might be still something inside our event buffer */
-	if (ebufcnt) {
-		if (__copy_to_user(&events[eventcnt], eventbuf,
-				   ebufcnt * sizeof(struct pollfd)))
-			eventcnt = -EFAULT;
-		else
-			eventcnt += ebufcnt;
+		if (ndpi < maxdpi)
+			break;
 	}
 
+	up_read(&epsem);
+
 	return eventcnt;
 }
 
@@ -1239,8 +1372,8 @@ static int __init eventpoll_init(void)
 	/* Initialize the list that will link "struct eventpoll" */
 	INIT_LIST_HEAD(&eplist);
 
-	/* Initialize the rwlock used to access "eplist" */
-	rwlock_init(&eplock);
+	/* Initialize the rwsem used to access "eplist" */
+	init_rwsem(&epsem);
 
 	/* Allocates slab cache used to allocate "struct epitem" items */
 	error = -ENOMEM;

fs/select.c

@@ -54,7 +54,7 @@ struct poll_table_page {
  * poll table.
  */
 
-void poll_freewait(poll_table* pt)
+void __pollfreewait(poll_table* pt)
 {
 	struct poll_table_page * p = pt->table;
 	while (p) {

include/linux/poll.h

@@ -14,14 +14,17 @@ struct poll_table_page;
 struct poll_table_struct;
 
 typedef void (*poll_queue_proc)(struct file *, wait_queue_head_t *, struct poll_table_struct *);
+typedef void (*poll_free_proc)(struct poll_table_struct *);
 
 typedef struct poll_table_struct {
 	poll_queue_proc qproc;
+	poll_free_proc fproc;
 	int error;
 	struct poll_table_page * table;
 } poll_table;
 
 extern void __pollwait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p);
+extern void __pollfreewait(poll_table* pt);
 
 static inline void poll_wait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p)
 {
@@ -29,9 +32,10 @@ static inline void poll_wait(struct file * filp, wait_queue_head_t * wait_addres
 		p->qproc(filp, wait_address, p);
 }
 
-static inline void poll_initwait_ex(poll_table* pt, poll_queue_proc qproc)
+static inline void poll_initwait_ex(poll_table* pt, poll_queue_proc qproc, poll_free_proc fproc)
 {
 	pt->qproc = qproc;
+	pt->fproc = fproc;
 	pt->error = 0;
 	pt->table = NULL;
 }
@@ -39,10 +43,15 @@ static inline void poll_initwait_ex(poll_table* pt, poll_queue_proc qproc)
 static inline void poll_initwait(poll_table* pt)
 {
 
-	poll_initwait_ex(pt, __pollwait);
+	poll_initwait_ex(pt, __pollwait, __pollfreewait);
 }
 
-extern void poll_freewait(poll_table* pt);
+static inline void poll_freewait(poll_table* pt)
+{
+
+	if (pt && pt->fproc)
+		pt->fproc(pt);
+}
 
 
 /*

kernel/ksyms.c

@@ -269,7 +269,7 @@ EXPORT_SYMBOL(generic_file_llseek);
 EXPORT_SYMBOL(remote_llseek);
 EXPORT_SYMBOL(no_llseek);
 EXPORT_SYMBOL(__pollwait);
-EXPORT_SYMBOL(poll_freewait);
+EXPORT_SYMBOL(__pollfreewait);
 EXPORT_SYMBOL(ROOT_DEV);
 EXPORT_SYMBOL(find_get_page);
 EXPORT_SYMBOL(find_lock_page);