Commit 41f0377f authored by Rusty Russell's avatar Rusty Russell

virtio: support unlocked queue kick

Based on patch by Christoph for virtio_blk speedup:

	Split virtqueue_kick to be able to do the actual notification
	outside the lock protecting the virtqueue.  This patch was
	originally done by Stefan Hajnoczi, but I can't find the
	original one anymore and had to recreated it from memory.
	Pointers to the original or corrections for the commit message
	are welcome.

Stefan's patch was here:

	https://github.com/stefanha/linux/commit/a6d06644e3a58e57a774e77d7dc34c4a5a2e7496
	http://www.spinics.net/lists/linux-virtualization/msg14616.html

Third time's the charm!
Signed-off-by: default avatarRusty Russell <rusty@rustcorp.com.au>
parent f96fde41
...@@ -262,19 +262,22 @@ int virtqueue_add_buf(struct virtqueue *_vq, ...@@ -262,19 +262,22 @@ int virtqueue_add_buf(struct virtqueue *_vq,
EXPORT_SYMBOL_GPL(virtqueue_add_buf); EXPORT_SYMBOL_GPL(virtqueue_add_buf);
/** /**
* virtqueue_kick - update after add_buf * virtqueue_kick_prepare - first half of split virtqueue_kick call.
* @vq: the struct virtqueue * @vq: the struct virtqueue
* *
* After one or more virtqueue_add_buf calls, invoke this to kick * Instead of virtqueue_kick(), you can do:
* the other side. * if (virtqueue_kick_prepare(vq))
* virtqueue_notify(vq);
* *
* Caller must ensure we don't call this with other virtqueue * This is sometimes useful because the virtqueue_kick_prepare() needs
* operations at the same time (except where noted). * to be serialized, but the actual virtqueue_notify() call does not.
*/ */
void virtqueue_kick(struct virtqueue *_vq) bool virtqueue_kick_prepare(struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); struct vring_virtqueue *vq = to_vvq(_vq);
u16 new, old; u16 new, old;
bool needs_kick;
START_USE(vq); START_USE(vq);
/* Descriptors and available array need to be set before we expose the /* Descriptors and available array need to be set before we expose the
* new available array entries. */ * new available array entries. */
...@@ -287,13 +290,46 @@ void virtqueue_kick(struct virtqueue *_vq) ...@@ -287,13 +290,46 @@ void virtqueue_kick(struct virtqueue *_vq)
/* Need to update avail index before checking if we should notify */ /* Need to update avail index before checking if we should notify */
virtio_mb(vq); virtio_mb(vq);
if (vq->event ? if (vq->event) {
vring_need_event(vring_avail_event(&vq->vring), new, old) : needs_kick = vring_need_event(vring_avail_event(&vq->vring),
!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY)) new, old);
/* Prod other side to tell it about changes. */ } else {
vq->notify(&vq->vq); needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
}
END_USE(vq); END_USE(vq);
return needs_kick;
}
EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
/**
* virtqueue_notify - second half of split virtqueue_kick call.
* @vq: the struct virtqueue
*
* This does not need to be serialized.
*/
void virtqueue_notify(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
/* Prod other side to tell it about changes. */
vq->notify(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_notify);
/**
* virtqueue_kick - update after add_buf
* @vq: the struct virtqueue
*
* After one or more virtqueue_add_buf calls, invoke this to kick
* the other side.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*/
void virtqueue_kick(struct virtqueue *vq)
{
if (virtqueue_kick_prepare(vq))
virtqueue_notify(vq);
} }
EXPORT_SYMBOL_GPL(virtqueue_kick); EXPORT_SYMBOL_GPL(virtqueue_kick);
......
...@@ -34,6 +34,10 @@ int virtqueue_add_buf(struct virtqueue *vq, ...@@ -34,6 +34,10 @@ int virtqueue_add_buf(struct virtqueue *vq,
void virtqueue_kick(struct virtqueue *vq); void virtqueue_kick(struct virtqueue *vq);
bool virtqueue_kick_prepare(struct virtqueue *vq);
void virtqueue_notify(struct virtqueue *vq);
void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len); void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len);
void virtqueue_disable_cb(struct virtqueue *vq); void virtqueue_disable_cb(struct virtqueue *vq);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment