Commit d57ed95d authored by Michael S. Tsirkin's avatar Michael S. Tsirkin Committed by Rusty Russell

virtio: use smp_XX barriers on SMP

virtio is communicating with a virtual "device" that actually runs on
another host processor. Thus SMP barriers can be used to control
memory access ordering.

Where possible, we should use SMP barriers which are more lightweight than
mandatory barriers, because mandatory barriers also control MMIO effects on
accesses through relaxed memory I/O windows (which virtio does not use)
(compare specifically smp_rmb and rmb on x86_64).

We can't just use smp_mb and friends though, because
we must force memory ordering even if guest is UP since host could be
running on another CPU, but SMP barriers are defined to barrier() in
that configuration. So, for UP fall back to mandatory barriers instead.
Signed-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
Signed-off-by: default avatarRusty Russell <rusty@rustcorp.com.au>
parent 97a545ab
...@@ -21,6 +21,24 @@ ...@@ -21,6 +21,24 @@
#include <linux/virtio_config.h> #include <linux/virtio_config.h>
#include <linux/device.h> #include <linux/device.h>
/* virtio guest is communicating with a virtual "device" that actually runs on
* a host processor. Memory barriers are used to control SMP effects. */
#ifdef CONFIG_SMP
/* Where possible, use SMP barriers which are more lightweight than mandatory
* barriers, because mandatory barriers control MMIO effects on accesses
* through relaxed memory I/O windows (which virtio does not use). */
#define virtio_mb() smp_mb()
#define virtio_rmb() smp_rmb()
#define virtio_wmb() smp_wmb()
#else
/* We must force memory ordering even if guest is UP since host could be
* running on another CPU, but SMP barriers are defined to barrier() in that
* configuration. So fall back to mandatory barriers instead. */
#define virtio_mb() mb()
#define virtio_rmb() rmb()
#define virtio_wmb() wmb()
#endif
#ifdef DEBUG #ifdef DEBUG
/* For development, we want to crash whenever the ring is screwed. */ /* For development, we want to crash whenever the ring is screwed. */
#define BAD_RING(_vq, fmt, args...) \ #define BAD_RING(_vq, fmt, args...) \
...@@ -220,13 +238,13 @@ static void vring_kick(struct virtqueue *_vq) ...@@ -220,13 +238,13 @@ static void vring_kick(struct virtqueue *_vq)
START_USE(vq); START_USE(vq);
/* Descriptors and available array need to be set before we expose the /* Descriptors and available array need to be set before we expose the
* new available array entries. */ * new available array entries. */
wmb(); virtio_wmb();
vq->vring.avail->idx += vq->num_added; vq->vring.avail->idx += vq->num_added;
vq->num_added = 0; vq->num_added = 0;
/* Need to update avail index before checking if we should notify */ /* Need to update avail index before checking if we should notify */
mb(); virtio_mb();
if (!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY)) if (!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
/* Prod other side to tell it about changes. */ /* Prod other side to tell it about changes. */
...@@ -285,7 +303,7 @@ static void *vring_get_buf(struct virtqueue *_vq, unsigned int *len) ...@@ -285,7 +303,7 @@ static void *vring_get_buf(struct virtqueue *_vq, unsigned int *len)
} }
/* Only get used array entries after they have been exposed by host. */ /* Only get used array entries after they have been exposed by host. */
rmb(); virtio_rmb();
i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id; i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
*len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len; *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
...@@ -323,7 +341,7 @@ static bool vring_enable_cb(struct virtqueue *_vq) ...@@ -323,7 +341,7 @@ static bool vring_enable_cb(struct virtqueue *_vq)
/* We optimistically turn back on interrupts, then check if there was /* We optimistically turn back on interrupts, then check if there was
* more to do. */ * more to do. */
vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
mb(); virtio_mb();
if (unlikely(more_used(vq))) { if (unlikely(more_used(vq))) {
END_USE(vq); END_USE(vq);
return false; return false;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment