Commit 8ccd54fe authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Pull virtio updates from Michael Tsirkin:
 "virtio,vhost,vdpa: features, fixes, and cleanups:

   - reduction in interrupt rate in virtio

   - perf improvement for VDUSE

   - scalability for vhost-scsi

   - non power of 2 ring support for packed rings

   - better management for mlx5 vdpa

   - suspend for snet

   - VIRTIO_F_NOTIFICATION_DATA

   - shared backend with vdpa-sim-blk

   - user VA support in vdpa-sim

   - better struct packing for virtio

  and fixes, cleanups all over the place"

* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (52 commits)
  vhost_vdpa: fix unmap process in no-batch mode
  MAINTAINERS: make me a reviewer of VIRTIO CORE AND NET DRIVERS
  tools/virtio: fix build caused by virtio_ring changes
  virtio_ring: add a struct device forward declaration
  vdpa_sim_blk: support shared backend
  vdpa_sim: move buffer allocation in the devices
  vdpa/snet: use likely/unlikely macros in hot functions
  vdpa/snet: implement kick_vq_with_data callback
  virtio-vdpa: add VIRTIO_F_NOTIFICATION_DATA feature support
  virtio: add VIRTIO_F_NOTIFICATION_DATA feature support
  vdpa/snet: support the suspend vDPA callback
  vdpa/snet: support getting and setting VQ state
  MAINTAINERS: add vringh.h to Virtio Core and Net Drivers
  vringh: address kdoc warnings
  vdpa: address kdoc warnings
  virtio_ring: don't update event idx on get_buf
  vdpa_sim: add support for user VA
  vdpa_sim: replace the spinlock with a mutex to protect the state
  vdpa_sim: use kthread worker
  vdpa_sim: make devices agnostic for work management
  ...
parents 0835b5ee c82729e0
...@@ -22212,6 +22212,7 @@ F: include/uapi/linux/virtio_console.h ...@@ -22212,6 +22212,7 @@ F: include/uapi/linux/virtio_console.h
VIRTIO CORE AND NET DRIVERS VIRTIO CORE AND NET DRIVERS
M: "Michael S. Tsirkin" <mst@redhat.com> M: "Michael S. Tsirkin" <mst@redhat.com>
M: Jason Wang <jasowang@redhat.com> M: Jason Wang <jasowang@redhat.com>
R: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
L: virtualization@lists.linux-foundation.org L: virtualization@lists.linux-foundation.org
S: Maintained S: Maintained
F: Documentation/ABI/testing/sysfs-bus-vdpa F: Documentation/ABI/testing/sysfs-bus-vdpa
...@@ -22225,6 +22226,7 @@ F: drivers/vdpa/ ...@@ -22225,6 +22226,7 @@ F: drivers/vdpa/
F: drivers/virtio/ F: drivers/virtio/
F: include/linux/vdpa.h F: include/linux/vdpa.h
F: include/linux/virtio*.h F: include/linux/virtio*.h
F: include/linux/vringh.h
F: include/uapi/linux/virtio_*.h F: include/uapi/linux/virtio_*.h
F: tools/virtio/ F: tools/virtio/
......
...@@ -391,7 +391,7 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev, ...@@ -391,7 +391,7 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev,
ccw_device_dma_free(vcdev->cdev, thinint_area, sizeof(*thinint_area)); ccw_device_dma_free(vcdev->cdev, thinint_area, sizeof(*thinint_area));
} }
static bool virtio_ccw_kvm_notify(struct virtqueue *vq) static inline bool virtio_ccw_do_kvm_notify(struct virtqueue *vq, u32 data)
{ {
struct virtio_ccw_vq_info *info = vq->priv; struct virtio_ccw_vq_info *info = vq->priv;
struct virtio_ccw_device *vcdev; struct virtio_ccw_device *vcdev;
...@@ -402,12 +402,22 @@ static bool virtio_ccw_kvm_notify(struct virtqueue *vq) ...@@ -402,12 +402,22 @@ static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
BUILD_BUG_ON(sizeof(struct subchannel_id) != sizeof(unsigned int)); BUILD_BUG_ON(sizeof(struct subchannel_id) != sizeof(unsigned int));
info->cookie = kvm_hypercall3(KVM_S390_VIRTIO_CCW_NOTIFY, info->cookie = kvm_hypercall3(KVM_S390_VIRTIO_CCW_NOTIFY,
*((unsigned int *)&schid), *((unsigned int *)&schid),
vq->index, info->cookie); data, info->cookie);
if (info->cookie < 0) if (info->cookie < 0)
return false; return false;
return true; return true;
} }
static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
{
return virtio_ccw_do_kvm_notify(vq, vq->index);
}
static bool virtio_ccw_kvm_notify_with_data(struct virtqueue *vq)
{
return virtio_ccw_do_kvm_notify(vq, vring_notification_data(vq));
}
static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev, static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev,
struct ccw1 *ccw, int index) struct ccw1 *ccw, int index)
{ {
...@@ -495,6 +505,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, ...@@ -495,6 +505,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
struct ccw1 *ccw) struct ccw1 *ccw)
{ {
struct virtio_ccw_device *vcdev = to_vc_device(vdev); struct virtio_ccw_device *vcdev = to_vc_device(vdev);
bool (*notify)(struct virtqueue *vq);
int err; int err;
struct virtqueue *vq = NULL; struct virtqueue *vq = NULL;
struct virtio_ccw_vq_info *info; struct virtio_ccw_vq_info *info;
...@@ -502,6 +513,11 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, ...@@ -502,6 +513,11 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
unsigned long flags; unsigned long flags;
bool may_reduce; bool may_reduce;
if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = virtio_ccw_kvm_notify_with_data;
else
notify = virtio_ccw_kvm_notify;
/* Allocate queue. */ /* Allocate queue. */
info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL); info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL);
if (!info) { if (!info) {
...@@ -524,7 +540,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, ...@@ -524,7 +540,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
may_reduce = vcdev->revision > 0; may_reduce = vcdev->revision > 0;
vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN, vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN,
vdev, true, may_reduce, ctx, vdev, true, may_reduce, ctx,
virtio_ccw_kvm_notify, callback, name); notify, callback, name);
if (!vq) { if (!vq) {
/* For now, we fail if we can't get the requested size. */ /* For now, we fail if we can't get the requested size. */
......
This diff is collapsed.
# SPDX-License-Identifier: GPL-2.0 # SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_SNET_VDPA) += snet_vdpa.o obj-$(CONFIG_SNET_VDPA) += snet_vdpa.o
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_main.o snet_vdpa-$(CONFIG_SNET_VDPA) += snet_main.o
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_ctrl.o
ifdef CONFIG_HWMON ifdef CONFIG_HWMON
snet_vdpa-$(CONFIG_SNET_VDPA) += snet_hwmon.o snet_vdpa-$(CONFIG_SNET_VDPA) += snet_hwmon.o
endif endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* SolidRun DPU driver for control plane
*
* Copyright (C) 2022-2023 SolidRun
*
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
*
*/
#include <linux/iopoll.h>
#include "snet_vdpa.h"
enum snet_ctrl_opcodes {
SNET_CTRL_OP_DESTROY = 1,
SNET_CTRL_OP_READ_VQ_STATE,
SNET_CTRL_OP_SUSPEND,
};
#define SNET_CTRL_TIMEOUT 2000000
#define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF
#define SNET_CTRL_IN_PROCESS_MASK 0x00010000
#define SNET_CTRL_CHUNK_RDY_MASK 0x00020000
#define SNET_CTRL_ERROR_MASK 0x0FFC0000
#define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
#define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \
!((val) & SNET_CTRL_IN_PROCESS_MASK))
#define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
/* Control register used to read data from the DPU */
struct snet_ctrl_reg_ctrl {
/* Chunk size in 4B words */
u16 data_size;
/* We are in the middle of a command */
u16 in_process:1;
/* A data chunk is ready and can be consumed */
u16 chunk_ready:1;
/* Error code */
u16 error:10;
/* Saved for future usage */
u16 rsvd:4;
};
/* Opcode register */
struct snet_ctrl_reg_op {
u16 opcode;
/* Only if VQ index is relevant for the command */
u16 vq_idx;
};
struct snet_ctrl_regs {
struct snet_ctrl_reg_op op;
struct snet_ctrl_reg_ctrl ctrl;
u32 rsvd;
u32 data[];
};
static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
{
return snet->bar + snet->psnet->cfg.ctrl_off;
}
static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
SNET_CTRL_TIMEOUT);
}
static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
}
static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
{
u32 val;
return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
SNET_CTRL_TIMEOUT);
}
static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
{
return ioread32(&ctrl_regs->data[word_idx]);
}
static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
{
return ioread32(&ctrl_regs->ctrl);
}
static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
iowrite32(val, &ctrl_regs->ctrl);
}
static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
iowrite32(val, &ctrl_regs->op);
}
static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
{
/* Wait until the DPU finishes completely.
* It will clear the opcode register.
*/
return snet_wait_for_empty_op(ctrl_regs);
}
/* Reading ctrl from the DPU:
* buf_size must be 4B aligned
*
* Steps:
*
* (1) Verify that the DPU is not in the middle of another operation by
* reading the in_process and error bits in the control register.
* (2) Write the request opcode and the VQ idx in the opcode register
* and write the buffer size in the control register.
* (3) Start readind chunks of data, chunk_ready bit indicates that a
* data chunk is available, we signal that we read the data by clearing the bit.
* (4) Detect that the transfer is completed when the in_process bit
* in the control register is cleared or when the an error appears.
*/
static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
u32 buf_size)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
u32 *bfr_ptr = (u32 *)buffer;
u32 val;
u16 buf_words;
int ret;
u16 words, i, tot_words = 0;
/* Supported for config 2+ */
if (!SNET_CFG_VER(snet, 2))
return -EOPNOTSUPP;
if (!IS_ALIGNED(buf_size, 4))
return -EINVAL;
mutex_lock(&snet->ctrl_lock);
buf_words = buf_size / 4;
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
goto exit;
}
/* We need to write the buffer size in the control register, and the opcode + vq index in
* the opcode register.
* We use a spinlock to serialize the writes.
*/
spin_lock(&snet->ctrl_spinlock);
snet_write_ctrl(regs, buf_words);
snet_write_op(regs, opcode | (vq_idx << 16));
spin_unlock(&snet->ctrl_spinlock);
while (buf_words != tot_words) {
ret = snet_wait_for_data(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for control data\n");
goto exit;
}
val = snet_read_ctrl(regs);
/* Error? */
if (val & SNET_CTRL_ERROR_MASK) {
ret = SNET_VAL_TO_ERR(val);
SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
goto exit;
}
words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
for (i = 0; i < words; i++) {
*bfr_ptr = snet_read32_word(regs, i);
bfr_ptr++;
}
tot_words += words;
/* Is the job completed? */
if (!(val & SNET_CTRL_IN_PROCESS_MASK))
break;
/* Clear the chunk ready bit and continue */
val &= ~SNET_CTRL_CHUNK_RDY_MASK;
snet_write_ctrl(regs, val);
}
ret = snet_wait_for_dpu_completion(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
/* Send a control message to the DPU using the old mechanism
* used with config version 1.
*/
static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
int ret;
mutex_lock(&snet->ctrl_lock);
/* Old mechanism uses just 1 register, the opcode register.
* Make sure that the opcode register is empty, and that the DPU isn't
* processing an old message.
*/
ret = snet_wait_for_empty_op(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
goto exit;
}
/* Write the message */
snet_write_op(regs, opcode);
/* DPU ACKs the message by clearing the opcode register */
ret = snet_wait_for_empty_op(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
/* Send a control message to the DPU.
* A control message is a message without payload.
*/
static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
{
struct pci_dev *pdev = snet->pdev;
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
u32 val;
int ret;
/* If config version is not 2+, use the old mechanism */
if (!SNET_CFG_VER(snet, 2))
return snet_send_ctrl_msg_old(snet, opcode);
mutex_lock(&snet->ctrl_lock);
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
goto exit;
}
/* We need to clear the control register and write the opcode + vq index in the opcode
* register.
* We use a spinlock to serialize the writes.
*/
spin_lock(&snet->ctrl_spinlock);
snet_write_ctrl(regs, 0);
snet_write_op(regs, opcode | (vq_idx << 16));
spin_unlock(&snet->ctrl_spinlock);
/* The DPU ACKs control messages by setting the chunk ready bit
* without data.
*/
ret = snet_wait_for_data(regs);
if (ret) {
SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
goto exit;
}
/* Check for errors */
val = snet_read_ctrl(regs);
ret = SNET_VAL_TO_ERR(val);
/* Clear the chunk ready bit */
val &= ~SNET_CTRL_CHUNK_RDY_MASK;
snet_write_ctrl(regs, val);
ret = snet_wait_for_dpu_completion(regs);
if (ret)
SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
ret);
exit:
mutex_unlock(&snet->ctrl_lock);
return ret;
}
void snet_ctrl_clear(struct snet *snet)
{
struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
snet_write_op(regs, 0);
}
int snet_destroy_dev(struct snet *snet)
{
return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
}
int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
{
return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
sizeof(*state));
}
int snet_suspend_dev(struct snet *snet)
{
return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
}
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
/* /*
* SolidRun DPU driver for control plane * SolidRun DPU driver for control plane
* *
* Copyright (C) 2022 SolidRun * Copyright (C) 2022-2023 SolidRun
* *
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com> * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
* *
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
/* /*
* SolidRun DPU driver for control plane * SolidRun DPU driver for control plane
* *
* Copyright (C) 2022 SolidRun * Copyright (C) 2022-2023 SolidRun
* *
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com> * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
* *
...@@ -16,14 +16,12 @@ ...@@ -16,14 +16,12 @@
/* SNET signature */ /* SNET signature */
#define SNET_SIGNATURE 0xD0D06363 #define SNET_SIGNATURE 0xD0D06363
/* Max. config version that we can work with */ /* Max. config version that we can work with */
#define SNET_CFG_VERSION 0x1 #define SNET_CFG_VERSION 0x2
/* Queue align */ /* Queue align */
#define SNET_QUEUE_ALIGNMENT PAGE_SIZE #define SNET_QUEUE_ALIGNMENT PAGE_SIZE
/* Kick value to notify that new data is available */ /* Kick value to notify that new data is available */
#define SNET_KICK_VAL 0x1 #define SNET_KICK_VAL 0x1
#define SNET_CONFIG_OFF 0x0 #define SNET_CONFIG_OFF 0x0
/* ACK timeout for a message */
#define SNET_ACK_TIMEOUT 2000000
/* How long we are willing to wait for a SNET device */ /* How long we are willing to wait for a SNET device */
#define SNET_DETECT_TIMEOUT 5000000 #define SNET_DETECT_TIMEOUT 5000000
/* How long should we wait for the DPU to read our config */ /* How long should we wait for the DPU to read our config */
...@@ -32,61 +30,16 @@ ...@@ -32,61 +30,16 @@
#define SNET_GENERAL_CFG_LEN 36 #define SNET_GENERAL_CFG_LEN 36
#define SNET_GENERAL_CFG_VQ_LEN 40 #define SNET_GENERAL_CFG_VQ_LEN 40
enum snet_msg {
SNET_MSG_DESTROY = 1,
};
static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
{ {
return container_of(vdpa, struct snet, vdpa); return container_of(vdpa, struct snet, vdpa);
} }
static int snet_wait_for_msg_ack(struct snet *snet)
{
struct pci_dev *pdev = snet->pdev;
int ret;
u32 val;
/* The DPU will clear the messages offset once messages
* are processed.
*/
ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.msg_off,
val, !val, 10, SNET_ACK_TIMEOUT);
if (ret)
SNET_WARN(pdev, "Timeout waiting for message ACK\n");
return ret;
}
/* Sends a message to the DPU.
* If blocking is set, the function will return once the
* message was processed by the DPU (or timeout).
*/
static int snet_send_msg(struct snet *snet, u32 msg, bool blocking)
{
int ret = 0;
/* Make sure the DPU acked last message before issuing a new one */
ret = snet_wait_for_msg_ack(snet);
if (ret)
return ret;
/* Write the message */
snet_write32(snet, snet->psnet->cfg.msg_off, msg);
if (blocking)
ret = snet_wait_for_msg_ack(snet);
else /* If non-blocking, flush the write by issuing a read */
snet_read32(snet, snet->psnet->cfg.msg_off);
return ret;
}
static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
{ {
struct snet *snet = data; struct snet *snet = data;
/* Call callback if any */ /* Call callback if any */
if (snet->cb.callback) if (likely(snet->cb.callback))
return snet->cb.callback(snet->cb.private); return snet->cb.callback(snet->cb.private);
return IRQ_HANDLED; return IRQ_HANDLED;
...@@ -96,7 +49,7 @@ static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) ...@@ -96,7 +49,7 @@ static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
{ {
struct snet_vq *vq = data; struct snet_vq *vq = data;
/* Call callback if any */ /* Call callback if any */
if (vq->cb.callback) if (likely(vq->cb.callback))
return vq->cb.callback(vq->cb.private); return vq->cb.callback(vq->cb.private);
return IRQ_HANDLED; return IRQ_HANDLED;
...@@ -153,12 +106,24 @@ static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) ...@@ -153,12 +106,24 @@ static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
{ {
struct snet *snet = vdpa_to_snet(vdev); struct snet *snet = vdpa_to_snet(vdev);
/* not ready - ignore */ /* not ready - ignore */
if (!snet->vqs[idx]->ready) if (unlikely(!snet->vqs[idx]->ready))
return; return;
iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
} }
static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
{
struct snet *snet = vdpa_to_snet(vdev);
u16 idx = data & 0xFFFF;
/* not ready - ignore */
if (unlikely(!snet->vqs[idx]->ready))
return;
iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
}
static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
{ {
struct snet *snet = vdpa_to_snet(vdev); struct snet *snet = vdpa_to_snet(vdev);
...@@ -181,33 +146,48 @@ static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) ...@@ -181,33 +146,48 @@ static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
return snet->vqs[idx]->ready; return snet->vqs[idx]->ready;
} }
static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
{ {
struct snet *snet = vdpa_to_snet(vdev);
/* Setting the VQ state is not supported.
* If the asked state is the same as the initial one
* we can ignore it.
*/
if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
const struct vdpa_vq_state_packed *p = &state->packed; const struct vdpa_vq_state_packed *p = &state->packed;
if (p->last_avail_counter == 1 && p->last_used_counter == 1 && if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
p->last_avail_idx == 0 && p->last_used_idx == 0) p->last_avail_idx == 0 && p->last_used_idx == 0)
return 0; return true;
} else { } else {
const struct vdpa_vq_state_split *s = &state->split; const struct vdpa_vq_state_split *s = &state->split;
if (s->avail_index == 0) if (s->avail_index == 0)
return true;
}
return false;
}
static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
{
struct snet *snet = vdpa_to_snet(vdev);
/* We can set any state for config version 2+ */
if (SNET_CFG_VER(snet, 2)) {
memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
return 0; return 0;
} }
/* Older config - we can't set the VQ state.
* Return 0 only if this is the initial state we use in the DPU.
*/
if (snet_vq_state_is_initial(snet, state))
return 0;
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
{ {
/* Not supported */ struct snet *snet = vdpa_to_snet(vdev);
return -EOPNOTSUPP;
return snet_read_vq_state(snet, idx, state);
} }
static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
...@@ -232,9 +212,9 @@ static int snet_reset_dev(struct snet *snet) ...@@ -232,9 +212,9 @@ static int snet_reset_dev(struct snet *snet)
if (!snet->status) if (!snet->status)
return 0; return 0;
/* If DPU started, send a destroy message */ /* If DPU started, destroy it */
if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
ret = snet_send_msg(snet, SNET_MSG_DESTROY, true); ret = snet_destroy_dev(snet);
/* Clear VQs */ /* Clear VQs */
for (i = 0; i < snet->cfg->vq_num; i++) { for (i = 0; i < snet->cfg->vq_num; i++) {
...@@ -258,7 +238,7 @@ static int snet_reset_dev(struct snet *snet) ...@@ -258,7 +238,7 @@ static int snet_reset_dev(struct snet *snet)
snet->dpu_ready = false; snet->dpu_ready = false;
if (ret) if (ret)
SNET_WARN(pdev, "Incomplete reset to SNET[%u] device\n", snet->sid); SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
else else
SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
...@@ -356,7 +336,7 @@ static int snet_write_conf(struct snet *snet) ...@@ -356,7 +336,7 @@ static int snet_write_conf(struct snet *snet)
* | DESC AREA | * | DESC AREA |
* | DEVICE AREA | * | DEVICE AREA |
* | DRIVER AREA | * | DRIVER AREA |
* | RESERVED | * | VQ STATE (CFG 2+) | RSVD |
* *
* Magic number should be written last, this is the DPU indication that the data is ready * Magic number should be written last, this is the DPU indication that the data is ready
*/ */
...@@ -391,12 +371,15 @@ static int snet_write_conf(struct snet *snet) ...@@ -391,12 +371,15 @@ static int snet_write_conf(struct snet *snet)
off += 8; off += 8;
snet_write64(snet, off, snet->vqs[i]->driver_area); snet_write64(snet, off, snet->vqs[i]->driver_area);
off += 8; off += 8;
/* Write VQ state if config version is 2+ */
if (SNET_CFG_VER(snet, 2))
snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
off += 4;
/* Ignore reserved */ /* Ignore reserved */
off += 8; off += 4;
} }
/* Clear snet messages address for this device */
snet_write32(snet, snet->psnet->cfg.msg_off, 0);
/* Write magic number - data is ready */ /* Write magic number - data is ready */
snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
...@@ -512,10 +495,25 @@ static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, ...@@ -512,10 +495,25 @@ static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
iowrite8(*buf_ptr++, cfg_ptr + i); iowrite8(*buf_ptr++, cfg_ptr + i);
} }
static int snet_suspend(struct vdpa_device *vdev)
{
struct snet *snet = vdpa_to_snet(vdev);
int ret;
ret = snet_suspend_dev(snet);
if (ret)
SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
else
SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
return ret;
}
static const struct vdpa_config_ops snet_config_ops = { static const struct vdpa_config_ops snet_config_ops = {
.set_vq_address = snet_set_vq_address, .set_vq_address = snet_set_vq_address,
.set_vq_num = snet_set_vq_num, .set_vq_num = snet_set_vq_num,
.kick_vq = snet_kick_vq, .kick_vq = snet_kick_vq,
.kick_vq_with_data = snet_kick_vq_with_data,
.set_vq_cb = snet_set_vq_cb, .set_vq_cb = snet_set_vq_cb,
.set_vq_ready = snet_set_vq_ready, .set_vq_ready = snet_set_vq_ready,
.get_vq_ready = snet_get_vq_ready, .get_vq_ready = snet_get_vq_ready,
...@@ -537,6 +535,7 @@ static const struct vdpa_config_ops snet_config_ops = { ...@@ -537,6 +535,7 @@ static const struct vdpa_config_ops snet_config_ops = {
.set_status = snet_set_status, .set_status = snet_set_status,
.get_config = snet_get_config, .get_config = snet_get_config,
.set_config = snet_set_config, .set_config = snet_set_config,
.suspend = snet_suspend,
}; };
static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
...@@ -697,7 +696,7 @@ static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) ...@@ -697,7 +696,7 @@ static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
off += 4; off += 4;
cfg->hwmon_off = psnet_read32(psnet, off); cfg->hwmon_off = psnet_read32(psnet, off);
off += 4; off += 4;
cfg->msg_off = psnet_read32(psnet, off); cfg->ctrl_off = psnet_read32(psnet, off);
off += 4; off += 4;
cfg->flags = psnet_read32(psnet, off); cfg->flags = psnet_read32(psnet, off);
off += 4; off += 4;
...@@ -997,6 +996,10 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev) ...@@ -997,6 +996,10 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
goto free_irqs; goto free_irqs;
} }
/* Init control mutex and spinlock */
mutex_init(&snet->ctrl_lock);
spin_lock_init(&snet->ctrl_spinlock);
/* Save pci device pointer */ /* Save pci device pointer */
snet->pdev = pdev; snet->pdev = pdev;
snet->psnet = psnet; snet->psnet = psnet;
...@@ -1013,6 +1016,9 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev) ...@@ -1013,6 +1016,9 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
/* Create a VirtIO config pointer */ /* Create a VirtIO config pointer */
snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
/* Clear control registers */
snet_ctrl_clear(snet);
pci_set_master(pdev); pci_set_master(pdev);
pci_set_drvdata(pdev, snet); pci_set_drvdata(pdev, snet);
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
/* /*
* SolidRun DPU driver for control plane * SolidRun DPU driver for control plane
* *
* Copyright (C) 2022 SolidRun * Copyright (C) 2022-2023 SolidRun
* *
* Author: Alvaro Karsz <alvaro.karsz@solid-run.com> * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
* *
...@@ -20,10 +20,15 @@ ...@@ -20,10 +20,15 @@
#define SNET_INFO(pdev, fmt, ...) dev_info(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__) #define SNET_INFO(pdev, fmt, ...) dev_info(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
#define SNET_DBG(pdev, fmt, ...) dev_dbg(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__) #define SNET_DBG(pdev, fmt, ...) dev_dbg(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
#define SNET_HAS_FEATURE(s, f) ((s)->negotiated_features & BIT_ULL(f)) #define SNET_HAS_FEATURE(s, f) ((s)->negotiated_features & BIT_ULL(f))
/* Check if negotiated config version is at least @ver */
#define SNET_CFG_VER(snet, ver) ((snet)->psnet->negotiated_cfg_ver >= (ver))
/* VQ struct */ /* VQ struct */
struct snet_vq { struct snet_vq {
/* VQ callback */ /* VQ callback */
struct vdpa_callback cb; struct vdpa_callback cb;
/* VQ state received from bus */
struct vdpa_vq_state vq_state;
/* desc base address */ /* desc base address */
u64 desc_area; u64 desc_area;
/* device base address */ /* device base address */
...@@ -51,6 +56,10 @@ struct snet { ...@@ -51,6 +56,10 @@ struct snet {
struct vdpa_device vdpa; struct vdpa_device vdpa;
/* Config callback */ /* Config callback */
struct vdpa_callback cb; struct vdpa_callback cb;
/* To lock the control mechanism */
struct mutex ctrl_lock;
/* Spinlock to protect critical parts in the control mechanism */
spinlock_t ctrl_spinlock;
/* array of virqueues */ /* array of virqueues */
struct snet_vq **vqs; struct snet_vq **vqs;
/* Used features */ /* Used features */
...@@ -117,8 +126,8 @@ struct snet_cfg { ...@@ -117,8 +126,8 @@ struct snet_cfg {
u32 kick_off; u32 kick_off;
/* Offset in PCI BAR for HW monitoring */ /* Offset in PCI BAR for HW monitoring */
u32 hwmon_off; u32 hwmon_off;
/* Offset in PCI BAR for SNET messages */ /* Offset in PCI BAR for Control mechanism */
u32 msg_off; u32 ctrl_off;
/* Config general flags - enum snet_cfg_flags */ /* Config general flags - enum snet_cfg_flags */
u32 flags; u32 flags;
/* Reserved for future usage */ /* Reserved for future usage */
...@@ -191,4 +200,9 @@ static inline void snet_write64(struct snet *snet, u32 off, u64 val) ...@@ -191,4 +200,9 @@ static inline void snet_write64(struct snet *snet, u32 off, u64 val)
void psnet_create_hwmon(struct pci_dev *pdev); void psnet_create_hwmon(struct pci_dev *pdev);
#endif #endif
void snet_ctrl_clear(struct snet *snet);
int snet_destroy_dev(struct snet *snet);
int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state);
int snet_suspend_dev(struct snet *snet);
#endif //_SNET_VDPA_H_ #endif //_SNET_VDPA_H_
This diff is collapsed.
...@@ -39,33 +39,34 @@ struct vdpasim_dev_attr { ...@@ -39,33 +39,34 @@ struct vdpasim_dev_attr {
u64 supported_features; u64 supported_features;
size_t alloc_size; size_t alloc_size;
size_t config_size; size_t config_size;
size_t buffer_size;
int nvqs; int nvqs;
u32 id; u32 id;
u32 ngroups; u32 ngroups;
u32 nas; u32 nas;
work_func_t work_fn; void (*work_fn)(struct vdpasim *vdpasim);
void (*get_config)(struct vdpasim *vdpasim, void *config); void (*get_config)(struct vdpasim *vdpasim, void *config);
void (*set_config)(struct vdpasim *vdpasim, const void *config); void (*set_config)(struct vdpasim *vdpasim, const void *config);
int (*get_stats)(struct vdpasim *vdpasim, u16 idx, int (*get_stats)(struct vdpasim *vdpasim, u16 idx,
struct sk_buff *msg, struct sk_buff *msg,
struct netlink_ext_ack *extack); struct netlink_ext_ack *extack);
void (*free)(struct vdpasim *vdpasim);
}; };
/* State of each vdpasim device */ /* State of each vdpasim device */
struct vdpasim { struct vdpasim {
struct vdpa_device vdpa; struct vdpa_device vdpa;
struct vdpasim_virtqueue *vqs; struct vdpasim_virtqueue *vqs;
struct work_struct work; struct kthread_worker *worker;
struct kthread_work work;
struct mm_struct *mm_bound;
struct vdpasim_dev_attr dev_attr; struct vdpasim_dev_attr dev_attr;
/* spinlock to synchronize virtqueue state */ /* mutex to synchronize virtqueue state */
spinlock_t lock; struct mutex mutex;
/* virtio config according to device type */ /* virtio config according to device type */
void *config; void *config;
struct vhost_iotlb *iommu; struct vhost_iotlb *iommu;
bool *iommu_pt; bool *iommu_pt;
void *buffer;
u32 status; u32 status;
u32 generation; u32 generation;
u64 features; u64 features;
...@@ -78,6 +79,7 @@ struct vdpasim { ...@@ -78,6 +79,7 @@ struct vdpasim {
struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr, struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
const struct vdpa_dev_set_config *config); const struct vdpa_dev_set_config *config);
void vdpasim_schedule_work(struct vdpasim *vdpasim);
/* TODO: cross-endian support */ /* TODO: cross-endian support */
static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim) static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim)
......
...@@ -11,7 +11,6 @@ ...@@ -11,7 +11,6 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/device.h> #include <linux/device.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/blkdev.h> #include <linux/blkdev.h>
#include <linux/vringh.h> #include <linux/vringh.h>
#include <linux/vdpa.h> #include <linux/vdpa.h>
...@@ -44,8 +43,39 @@ ...@@ -44,8 +43,39 @@
#define VDPASIM_BLK_AS_NUM 1 #define VDPASIM_BLK_AS_NUM 1
#define VDPASIM_BLK_GROUP_NUM 1 #define VDPASIM_BLK_GROUP_NUM 1
struct vdpasim_blk {
struct vdpasim vdpasim;
void *buffer;
bool shared_backend;
};
static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim)
{
return container_of(vdpasim, struct vdpasim_blk, vdpasim);
}
static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim"; static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
static bool shared_backend;
module_param(shared_backend, bool, 0444);
MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices");
static void *shared_buffer;
/* mutex to synchronize shared_buffer access */
static DEFINE_MUTEX(shared_buffer_mutex);
static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk)
{
if (blk->shared_backend)
mutex_lock(&shared_buffer_mutex);
}
static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk)
{
if (blk->shared_backend)
mutex_unlock(&shared_buffer_mutex);
}
static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
u64 num_sectors, u64 max_sectors) u64 num_sectors, u64 max_sectors)
{ {
...@@ -79,6 +109,7 @@ static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, ...@@ -79,6 +109,7 @@ static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
struct vdpasim_virtqueue *vq) struct vdpasim_virtqueue *vq)
{ {
struct vdpasim_blk *blk = sim_to_blk(vdpasim);
size_t pushed = 0, to_pull, to_push; size_t pushed = 0, to_pull, to_push;
struct virtio_blk_outhdr hdr; struct virtio_blk_outhdr hdr;
bool handled = false; bool handled = false;
...@@ -144,9 +175,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, ...@@ -144,9 +175,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
break; break;
} }
vdpasim_blk_buffer_lock(blk);
bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
vdpasim->buffer + offset, blk->buffer + offset, to_push);
to_push); vdpasim_blk_buffer_unlock(blk);
if (bytes < 0) { if (bytes < 0) {
dev_dbg(&vdpasim->vdpa.dev, dev_dbg(&vdpasim->vdpa.dev,
"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", "vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
...@@ -166,9 +198,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, ...@@ -166,9 +198,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
break; break;
} }
vdpasim_blk_buffer_lock(blk);
bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov,
vdpasim->buffer + offset, blk->buffer + offset, to_pull);
to_pull); vdpasim_blk_buffer_unlock(blk);
if (bytes < 0) { if (bytes < 0) {
dev_dbg(&vdpasim->vdpa.dev, dev_dbg(&vdpasim->vdpa.dev,
"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
...@@ -248,8 +281,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, ...@@ -248,8 +281,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
} }
if (type == VIRTIO_BLK_T_WRITE_ZEROES) { if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
memset(vdpasim->buffer + offset, 0, vdpasim_blk_buffer_lock(blk);
memset(blk->buffer + offset, 0,
num_sectors << SECTOR_SHIFT); num_sectors << SECTOR_SHIFT);
vdpasim_blk_buffer_unlock(blk);
} }
break; break;
...@@ -286,13 +321,12 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, ...@@ -286,13 +321,12 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
return handled; return handled;
} }
static void vdpasim_blk_work(struct work_struct *work) static void vdpasim_blk_work(struct vdpasim *vdpasim)
{ {
struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
bool reschedule = false; bool reschedule = false;
int i; int i;
spin_lock(&vdpasim->lock); mutex_lock(&vdpasim->mutex);
if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
goto out; goto out;
...@@ -323,10 +357,10 @@ static void vdpasim_blk_work(struct work_struct *work) ...@@ -323,10 +357,10 @@ static void vdpasim_blk_work(struct work_struct *work)
} }
} }
out: out:
spin_unlock(&vdpasim->lock); mutex_unlock(&vdpasim->mutex);
if (reschedule) if (reschedule)
schedule_work(&vdpasim->work); vdpasim_schedule_work(vdpasim);
} }
static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config) static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
...@@ -355,6 +389,14 @@ static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config) ...@@ -355,6 +389,14 @@ static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
} }
static void vdpasim_blk_free(struct vdpasim *vdpasim)
{
struct vdpasim_blk *blk = sim_to_blk(vdpasim);
if (!blk->shared_backend)
kvfree(blk->buffer);
}
static void vdpasim_blk_mgmtdev_release(struct device *dev) static void vdpasim_blk_mgmtdev_release(struct device *dev)
{ {
} }
...@@ -368,6 +410,7 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, ...@@ -368,6 +410,7 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
const struct vdpa_dev_set_config *config) const struct vdpa_dev_set_config *config)
{ {
struct vdpasim_dev_attr dev_attr = {}; struct vdpasim_dev_attr dev_attr = {};
struct vdpasim_blk *blk;
struct vdpasim *simdev; struct vdpasim *simdev;
int ret; int ret;
...@@ -378,16 +421,30 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, ...@@ -378,16 +421,30 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
dev_attr.nvqs = VDPASIM_BLK_VQ_NUM; dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM; dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
dev_attr.nas = VDPASIM_BLK_AS_NUM; dev_attr.nas = VDPASIM_BLK_AS_NUM;
dev_attr.alloc_size = sizeof(struct vdpasim); dev_attr.alloc_size = sizeof(struct vdpasim_blk);
dev_attr.config_size = sizeof(struct virtio_blk_config); dev_attr.config_size = sizeof(struct virtio_blk_config);
dev_attr.get_config = vdpasim_blk_get_config; dev_attr.get_config = vdpasim_blk_get_config;
dev_attr.work_fn = vdpasim_blk_work; dev_attr.work_fn = vdpasim_blk_work;
dev_attr.buffer_size = VDPASIM_BLK_CAPACITY << SECTOR_SHIFT; dev_attr.free = vdpasim_blk_free;
simdev = vdpasim_create(&dev_attr, config); simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev)) if (IS_ERR(simdev))
return PTR_ERR(simdev); return PTR_ERR(simdev);
blk = sim_to_blk(simdev);
blk->shared_backend = shared_backend;
if (blk->shared_backend) {
blk->buffer = shared_buffer;
} else {
blk->buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
GFP_KERNEL);
if (!blk->buffer) {
ret = -ENOMEM;
goto put_dev;
}
}
ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM); ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM);
if (ret) if (ret)
goto put_dev; goto put_dev;
...@@ -437,6 +494,15 @@ static int __init vdpasim_blk_init(void) ...@@ -437,6 +494,15 @@ static int __init vdpasim_blk_init(void)
if (ret) if (ret)
goto parent_err; goto parent_err;
if (shared_backend) {
shared_buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
GFP_KERNEL);
if (!shared_buffer) {
ret = -ENOMEM;
goto parent_err;
}
}
return 0; return 0;
parent_err: parent_err:
...@@ -446,6 +512,7 @@ static int __init vdpasim_blk_init(void) ...@@ -446,6 +512,7 @@ static int __init vdpasim_blk_init(void)
static void __exit vdpasim_blk_exit(void) static void __exit vdpasim_blk_exit(void)
{ {
kvfree(shared_buffer);
vdpa_mgmtdev_unregister(&mgmt_dev); vdpa_mgmtdev_unregister(&mgmt_dev);
device_unregister(&vdpasim_blk_mgmtdev); device_unregister(&vdpasim_blk_mgmtdev);
} }
......
...@@ -11,7 +11,6 @@ ...@@ -11,7 +11,6 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/device.h> #include <linux/device.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/etherdevice.h> #include <linux/etherdevice.h>
#include <linux/vringh.h> #include <linux/vringh.h>
#include <linux/vdpa.h> #include <linux/vdpa.h>
...@@ -59,6 +58,7 @@ struct vdpasim_net{ ...@@ -59,6 +58,7 @@ struct vdpasim_net{
struct vdpasim_dataq_stats tx_stats; struct vdpasim_dataq_stats tx_stats;
struct vdpasim_dataq_stats rx_stats; struct vdpasim_dataq_stats rx_stats;
struct vdpasim_cq_stats cq_stats; struct vdpasim_cq_stats cq_stats;
void *buffer;
}; };
static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim) static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim)
...@@ -88,14 +88,15 @@ static bool receive_filter(struct vdpasim *vdpasim, size_t len) ...@@ -88,14 +88,15 @@ static bool receive_filter(struct vdpasim *vdpasim, size_t len)
size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) : size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) :
sizeof(struct virtio_net_hdr); sizeof(struct virtio_net_hdr);
struct virtio_net_config *vio_config = vdpasim->config; struct virtio_net_config *vio_config = vdpasim->config;
struct vdpasim_net *net = sim_to_net(vdpasim);
if (len < ETH_ALEN + hdr_len) if (len < ETH_ALEN + hdr_len)
return false; return false;
if (is_broadcast_ether_addr(vdpasim->buffer + hdr_len) || if (is_broadcast_ether_addr(net->buffer + hdr_len) ||
is_multicast_ether_addr(vdpasim->buffer + hdr_len)) is_multicast_ether_addr(net->buffer + hdr_len))
return true; return true;
if (!strncmp(vdpasim->buffer + hdr_len, vio_config->mac, ETH_ALEN)) if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN))
return true; return true;
return false; return false;
...@@ -192,9 +193,8 @@ static void vdpasim_handle_cvq(struct vdpasim *vdpasim) ...@@ -192,9 +193,8 @@ static void vdpasim_handle_cvq(struct vdpasim *vdpasim)
u64_stats_update_end(&net->cq_stats.syncp); u64_stats_update_end(&net->cq_stats.syncp);
} }
static void vdpasim_net_work(struct work_struct *work) static void vdpasim_net_work(struct vdpasim *vdpasim)
{ {
struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
struct vdpasim_net *net = sim_to_net(vdpasim); struct vdpasim_net *net = sim_to_net(vdpasim);
...@@ -203,7 +203,7 @@ static void vdpasim_net_work(struct work_struct *work) ...@@ -203,7 +203,7 @@ static void vdpasim_net_work(struct work_struct *work)
u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0; u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0;
int err; int err;
spin_lock(&vdpasim->lock); mutex_lock(&vdpasim->mutex);
if (!vdpasim->running) if (!vdpasim->running)
goto out; goto out;
...@@ -227,8 +227,7 @@ static void vdpasim_net_work(struct work_struct *work) ...@@ -227,8 +227,7 @@ static void vdpasim_net_work(struct work_struct *work)
++tx_pkts; ++tx_pkts;
read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov, read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov,
vdpasim->buffer, net->buffer, PAGE_SIZE);
PAGE_SIZE);
tx_bytes += read; tx_bytes += read;
...@@ -247,7 +246,7 @@ static void vdpasim_net_work(struct work_struct *work) ...@@ -247,7 +246,7 @@ static void vdpasim_net_work(struct work_struct *work)
} }
write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov, write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov,
vdpasim->buffer, read); net->buffer, read);
if (write <= 0) { if (write <= 0) {
++rx_errors; ++rx_errors;
break; break;
...@@ -260,13 +259,13 @@ static void vdpasim_net_work(struct work_struct *work) ...@@ -260,13 +259,13 @@ static void vdpasim_net_work(struct work_struct *work)
vdpasim_net_complete(rxq, write); vdpasim_net_complete(rxq, write);
if (tx_pkts > 4) { if (tx_pkts > 4) {
schedule_work(&vdpasim->work); vdpasim_schedule_work(vdpasim);
goto out; goto out;
} }
} }
out: out:
spin_unlock(&vdpasim->lock); mutex_unlock(&vdpasim->mutex);
u64_stats_update_begin(&net->tx_stats.syncp); u64_stats_update_begin(&net->tx_stats.syncp);
net->tx_stats.pkts += tx_pkts; net->tx_stats.pkts += tx_pkts;
...@@ -429,6 +428,13 @@ static void vdpasim_net_setup_config(struct vdpasim *vdpasim, ...@@ -429,6 +428,13 @@ static void vdpasim_net_setup_config(struct vdpasim *vdpasim,
vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500); vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500);
} }
static void vdpasim_net_free(struct vdpasim *vdpasim)
{
struct vdpasim_net *net = sim_to_net(vdpasim);
kvfree(net->buffer);
}
static void vdpasim_net_mgmtdev_release(struct device *dev) static void vdpasim_net_mgmtdev_release(struct device *dev)
{ {
} }
...@@ -458,7 +464,7 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, ...@@ -458,7 +464,7 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
dev_attr.get_config = vdpasim_net_get_config; dev_attr.get_config = vdpasim_net_get_config;
dev_attr.work_fn = vdpasim_net_work; dev_attr.work_fn = vdpasim_net_work;
dev_attr.get_stats = vdpasim_net_get_stats; dev_attr.get_stats = vdpasim_net_get_stats;
dev_attr.buffer_size = PAGE_SIZE; dev_attr.free = vdpasim_net_free;
simdev = vdpasim_create(&dev_attr, config); simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev)) if (IS_ERR(simdev))
...@@ -472,6 +478,12 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, ...@@ -472,6 +478,12 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
u64_stats_init(&net->rx_stats.syncp); u64_stats_init(&net->rx_stats.syncp);
u64_stats_init(&net->cq_stats.syncp); u64_stats_init(&net->cq_stats.syncp);
net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL);
if (!net->buffer) {
ret = -ENOMEM;
goto reg_err;
}
/* /*
* Initialization must be completed before this call, since it can * Initialization must be completed before this call, since it can
* connect the device to the vDPA bus, so requests can arrive after * connect the device to the vDPA bus, so requests can arrive after
......
This diff is collapsed.
...@@ -229,7 +229,10 @@ struct vhost_scsi_ctx { ...@@ -229,7 +229,10 @@ struct vhost_scsi_ctx {
struct iov_iter out_iter; struct iov_iter out_iter;
}; };
/* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */ /*
* Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
* configfs management operations.
*/
static DEFINE_MUTEX(vhost_scsi_mutex); static DEFINE_MUTEX(vhost_scsi_mutex);
static LIST_HEAD(vhost_scsi_list); static LIST_HEAD(vhost_scsi_list);
...@@ -1501,7 +1504,7 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds) ...@@ -1501,7 +1504,7 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
* vhost_scsi_tpg with an active struct vhost_scsi_nexus * vhost_scsi_tpg with an active struct vhost_scsi_nexus
* *
* The lock nesting rule is: * The lock nesting rule is:
* vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex * vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
*/ */
static int static int
vhost_scsi_set_endpoint(struct vhost_scsi *vs, vhost_scsi_set_endpoint(struct vhost_scsi *vs,
...@@ -1515,7 +1518,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1515,7 +1518,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
int index, ret, i, len; int index, ret, i, len;
bool match = false; bool match = false;
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&vs->dev.mutex); mutex_lock(&vs->dev.mutex);
/* Verify that ring has been setup correctly. */ /* Verify that ring has been setup correctly. */
...@@ -1536,6 +1538,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1536,6 +1538,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (vs->vs_tpg) if (vs->vs_tpg)
memcpy(vs_tpg, vs->vs_tpg, len); memcpy(vs_tpg, vs->vs_tpg, len);
mutex_lock(&vhost_scsi_mutex);
list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) { list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
mutex_lock(&tpg->tv_tpg_mutex); mutex_lock(&tpg->tv_tpg_mutex);
if (!tpg->tpg_nexus) { if (!tpg->tpg_nexus) {
...@@ -1551,6 +1554,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1551,6 +1554,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) { if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) { if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
mutex_unlock(&tpg->tv_tpg_mutex); mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
ret = -EEXIST; ret = -EEXIST;
goto undepend; goto undepend;
} }
...@@ -1565,6 +1569,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1565,6 +1569,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
if (ret) { if (ret) {
pr_warn("target_depend_item() failed: %d\n", ret); pr_warn("target_depend_item() failed: %d\n", ret);
mutex_unlock(&tpg->tv_tpg_mutex); mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
goto undepend; goto undepend;
} }
tpg->tv_tpg_vhost_count++; tpg->tv_tpg_vhost_count++;
...@@ -1574,6 +1579,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1574,6 +1579,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
} }
mutex_unlock(&tpg->tv_tpg_mutex); mutex_unlock(&tpg->tv_tpg_mutex);
} }
mutex_unlock(&vhost_scsi_mutex);
if (match) { if (match) {
memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn, memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
...@@ -1629,7 +1635,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, ...@@ -1629,7 +1635,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
kfree(vs_tpg); kfree(vs_tpg);
out: out:
mutex_unlock(&vs->dev.mutex); mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return ret; return ret;
} }
...@@ -1645,7 +1650,6 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, ...@@ -1645,7 +1650,6 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
int index, ret, i; int index, ret, i;
u8 target; u8 target;
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&vs->dev.mutex); mutex_lock(&vs->dev.mutex);
/* Verify that ring has been setup correctly. */ /* Verify that ring has been setup correctly. */
for (index = 0; index < vs->dev.nvqs; ++index) { for (index = 0; index < vs->dev.nvqs; ++index) {
...@@ -1666,11 +1670,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, ...@@ -1666,11 +1670,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
if (!tpg) if (!tpg)
continue; continue;
mutex_lock(&tpg->tv_tpg_mutex);
tv_tport = tpg->tport; tv_tport = tpg->tport;
if (!tv_tport) { if (!tv_tport) {
ret = -ENODEV; ret = -ENODEV;
goto err_tpg; goto err_dev;
} }
if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) { if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
...@@ -1679,21 +1682,14 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, ...@@ -1679,21 +1682,14 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
tv_tport->tport_name, tpg->tport_tpgt, tv_tport->tport_name, tpg->tport_tpgt,
t->vhost_wwpn, t->vhost_tpgt); t->vhost_wwpn, t->vhost_tpgt);
ret = -EINVAL; ret = -EINVAL;
goto err_tpg; goto err_dev;
} }
tpg->tv_tpg_vhost_count--;
tpg->vhost_scsi = NULL;
vs->vs_tpg[target] = NULL;
match = true; match = true;
mutex_unlock(&tpg->tv_tpg_mutex);
/*
* Release se_tpg->tpg_group.cg_item configfs dependency now
* to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur.
*/
se_tpg = &tpg->se_tpg;
target_undepend_item(&se_tpg->tpg_group.cg_item);
} }
if (match) { if (!match)
goto free_vs_tpg;
/* Prevent new cmds from starting and accessing the tpgs/sessions */
for (i = 0; i < vs->dev.nvqs; i++) { for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq; vq = &vs->vqs[i].vq;
mutex_lock(&vq->mutex); mutex_lock(&vq->mutex);
...@@ -1707,7 +1703,30 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, ...@@ -1707,7 +1703,30 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
vq = &vs->vqs[i].vq; vq = &vs->vqs[i].vq;
vhost_scsi_destroy_vq_cmds(vq); vhost_scsi_destroy_vq_cmds(vq);
} }
/*
* We can now release our hold on the tpg and sessions and userspace
* can free them after this point.
*/
for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
target = i;
tpg = vs->vs_tpg[target];
if (!tpg)
continue;
mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_vhost_count--;
tpg->vhost_scsi = NULL;
vs->vs_tpg[target] = NULL;
mutex_unlock(&tpg->tv_tpg_mutex);
se_tpg = &tpg->se_tpg;
target_undepend_item(&se_tpg->tpg_group.cg_item);
} }
free_vs_tpg:
/* /*
* Act as synchronize_rcu to make sure access to * Act as synchronize_rcu to make sure access to
* old vs->vs_tpg is finished. * old vs->vs_tpg is finished.
...@@ -1717,14 +1736,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, ...@@ -1717,14 +1736,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
vs->vs_tpg = NULL; vs->vs_tpg = NULL;
WARN_ON(vs->vs_events_nr); WARN_ON(vs->vs_events_nr);
mutex_unlock(&vs->dev.mutex); mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return 0; return 0;
err_tpg:
mutex_unlock(&tpg->tv_tpg_mutex);
err_dev: err_dev:
mutex_unlock(&vs->dev.mutex); mutex_unlock(&vs->dev.mutex);
mutex_unlock(&vhost_scsi_mutex);
return ret; return ret;
} }
...@@ -1965,8 +1980,6 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg, ...@@ -1965,8 +1980,6 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
if (!vs) if (!vs)
return; return;
mutex_lock(&vs->dev.mutex);
if (plug) if (plug)
reason = VIRTIO_SCSI_EVT_RESET_RESCAN; reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
else else
...@@ -1974,11 +1987,18 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg, ...@@ -1974,11 +1987,18 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
mutex_lock(&vq->mutex); mutex_lock(&vq->mutex);
/*
* We can't queue events if the backend has been cleared, because
* we could end up queueing an event after the flush.
*/
if (!vhost_vq_get_backend(vq))
goto unlock;
if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG)) if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
vhost_scsi_send_evt(vs, tpg, lun, vhost_scsi_send_evt(vs, tpg, lun,
VIRTIO_SCSI_T_TRANSPORT_RESET, reason); VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
unlock:
mutex_unlock(&vq->mutex); mutex_unlock(&vq->mutex);
mutex_unlock(&vs->dev.mutex);
} }
static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun) static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
...@@ -1997,15 +2017,10 @@ static int vhost_scsi_port_link(struct se_portal_group *se_tpg, ...@@ -1997,15 +2017,10 @@ static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg *tpg = container_of(se_tpg,
struct vhost_scsi_tpg, se_tpg); struct vhost_scsi_tpg, se_tpg);
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&tpg->tv_tpg_mutex); mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_port_count++; tpg->tv_tpg_port_count++;
mutex_unlock(&tpg->tv_tpg_mutex);
vhost_scsi_hotplug(tpg, lun); vhost_scsi_hotplug(tpg, lun);
mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
return 0; return 0;
} }
...@@ -2016,15 +2031,10 @@ static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg, ...@@ -2016,15 +2031,10 @@ static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg *tpg = container_of(se_tpg,
struct vhost_scsi_tpg, se_tpg); struct vhost_scsi_tpg, se_tpg);
mutex_lock(&vhost_scsi_mutex);
mutex_lock(&tpg->tv_tpg_mutex); mutex_lock(&tpg->tv_tpg_mutex);
tpg->tv_tpg_port_count--; tpg->tv_tpg_port_count--;
mutex_unlock(&tpg->tv_tpg_mutex);
vhost_scsi_hotunplug(tpg, lun); vhost_scsi_hotunplug(tpg, lun);
mutex_unlock(&tpg->tv_tpg_mutex);
mutex_unlock(&vhost_scsi_mutex);
} }
static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store( static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
......
...@@ -219,6 +219,28 @@ static int vhost_vdpa_reset(struct vhost_vdpa *v) ...@@ -219,6 +219,28 @@ static int vhost_vdpa_reset(struct vhost_vdpa *v)
return vdpa_reset(vdpa); return vdpa_reset(vdpa);
} }
static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
if (!vdpa->use_va || !ops->bind_mm)
return 0;
return ops->bind_mm(vdpa, v->vdev.mm);
}
static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
if (!vdpa->use_va || !ops->unbind_mm)
return;
ops->unbind_mm(vdpa);
}
static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
{ {
struct vdpa_device *vdpa = v->vdpa; struct vdpa_device *vdpa = v->vdpa;
...@@ -599,9 +621,11 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, ...@@ -599,9 +621,11 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
if (vq->call_ctx.ctx) { if (vq->call_ctx.ctx) {
cb.callback = vhost_vdpa_virtqueue_cb; cb.callback = vhost_vdpa_virtqueue_cb;
cb.private = vq; cb.private = vq;
cb.trigger = vq->call_ctx.ctx;
} else { } else {
cb.callback = NULL; cb.callback = NULL;
cb.private = NULL; cb.private = NULL;
cb.trigger = NULL;
} }
ops->set_vq_cb(vdpa, idx, &cb); ops->set_vq_cb(vdpa, idx, &cb);
vhost_vdpa_setup_vq_irq(v, idx); vhost_vdpa_setup_vq_irq(v, idx);
...@@ -716,6 +740,17 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, ...@@ -716,6 +740,17 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
break; break;
} }
if (r)
goto out;
switch (cmd) {
case VHOST_SET_OWNER:
r = vhost_vdpa_bind_mm(v);
if (r)
vhost_dev_reset_owner(d, NULL);
break;
}
out:
mutex_unlock(&d->mutex); mutex_unlock(&d->mutex);
return r; return r;
} }
...@@ -851,11 +886,7 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, ...@@ -851,11 +886,7 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
if (!v->in_batch) if (!v->in_batch)
ops->set_map(vdpa, asid, iotlb); ops->set_map(vdpa, asid, iotlb);
} }
/* If we are in the middle of batch processing, delay the free
* of AS until BATCH_END.
*/
if (!v->in_batch && !iotlb->nmaps)
vhost_vdpa_remove_as(v, asid);
} }
static int vhost_vdpa_va_map(struct vhost_vdpa *v, static int vhost_vdpa_va_map(struct vhost_vdpa *v,
...@@ -1112,8 +1143,6 @@ static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid, ...@@ -1112,8 +1143,6 @@ static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
if (v->in_batch && ops->set_map) if (v->in_batch && ops->set_map)
ops->set_map(vdpa, asid, iotlb); ops->set_map(vdpa, asid, iotlb);
v->in_batch = false; v->in_batch = false;
if (!iotlb->nmaps)
vhost_vdpa_remove_as(v, asid);
break; break;
default: default:
r = -EINVAL; r = -EINVAL;
...@@ -1287,6 +1316,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) ...@@ -1287,6 +1316,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
vhost_vdpa_clean_irq(v); vhost_vdpa_clean_irq(v);
vhost_vdpa_reset(v); vhost_vdpa_reset(v);
vhost_dev_stop(&v->vdev); vhost_dev_stop(&v->vdev);
vhost_vdpa_unbind_mm(v);
vhost_vdpa_config_put(v); vhost_vdpa_config_put(v);
vhost_vdpa_cleanup(v); vhost_vdpa_cleanup(v);
mutex_unlock(&d->mutex); mutex_unlock(&d->mutex);
......
...@@ -434,8 +434,7 @@ static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, ...@@ -434,8 +434,7 @@ static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
size_t event __maybe_unused = size_t event __maybe_unused =
vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return sizeof(*vq->avail) + return size_add(struct_size(vq->avail, ring, num), event);
sizeof(*vq->avail->ring) * num + event;
} }
static size_t vhost_get_used_size(struct vhost_virtqueue *vq, static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
...@@ -444,8 +443,7 @@ static size_t vhost_get_used_size(struct vhost_virtqueue *vq, ...@@ -444,8 +443,7 @@ static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
size_t event __maybe_unused = size_t event __maybe_unused =
vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return sizeof(*vq->used) + return size_add(struct_size(vq->used, ring, num), event);
sizeof(*vq->used->ring) * num + event;
} }
static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
......
...@@ -636,9 +636,9 @@ static inline int xfer_to_user(const struct vringh *vrh, ...@@ -636,9 +636,9 @@ static inline int xfer_to_user(const struct vringh *vrh,
* @features: the feature bits for this ring. * @features: the feature bits for this ring.
* @num: the number of elements. * @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O. * @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer. * @desc: the userspace descriptor pointer.
* @avail: the userpace avail pointer. * @avail: the userspace avail pointer.
* @used: the userpace used pointer. * @used: the userspace used pointer.
* *
* Returns an error if num is invalid: you should check pointers * Returns an error if num is invalid: you should check pointers
* yourself! * yourself!
...@@ -911,9 +911,9 @@ static inline int kern_xfer(const struct vringh *vrh, void *dst, ...@@ -911,9 +911,9 @@ static inline int kern_xfer(const struct vringh *vrh, void *dst,
* @features: the feature bits for this ring. * @features: the feature bits for this ring.
* @num: the number of elements. * @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O. * @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer. * @desc: the userspace descriptor pointer.
* @avail: the userpace avail pointer. * @avail: the userspace avail pointer.
* @used: the userpace used pointer. * @used: the userspace used pointer.
* *
* Returns an error if num is invalid. * Returns an error if num is invalid.
*/ */
...@@ -1094,10 +1094,17 @@ EXPORT_SYMBOL(vringh_need_notify_kern); ...@@ -1094,10 +1094,17 @@ EXPORT_SYMBOL(vringh_need_notify_kern);
#if IS_REACHABLE(CONFIG_VHOST_IOTLB) #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
struct iotlb_vec {
union {
struct iovec *iovec;
struct bio_vec *bvec;
} iov;
size_t count;
};
static int iotlb_translate(const struct vringh *vrh, static int iotlb_translate(const struct vringh *vrh,
u64 addr, u64 len, u64 *translated, u64 addr, u64 len, u64 *translated,
struct bio_vec iov[], struct iotlb_vec *ivec, u32 perm)
int iov_size, u32 perm)
{ {
struct vhost_iotlb_map *map; struct vhost_iotlb_map *map;
struct vhost_iotlb *iotlb = vrh->iotlb; struct vhost_iotlb *iotlb = vrh->iotlb;
...@@ -1107,9 +1114,11 @@ static int iotlb_translate(const struct vringh *vrh, ...@@ -1107,9 +1114,11 @@ static int iotlb_translate(const struct vringh *vrh,
spin_lock(vrh->iotlb_lock); spin_lock(vrh->iotlb_lock);
while (len > s) { while (len > s) {
u64 size, pa, pfn; uintptr_t io_addr;
size_t io_len;
u64 size;
if (unlikely(ret >= iov_size)) { if (unlikely(ret >= ivec->count)) {
ret = -ENOBUFS; ret = -ENOBUFS;
break; break;
} }
...@@ -1124,10 +1133,22 @@ static int iotlb_translate(const struct vringh *vrh, ...@@ -1124,10 +1133,22 @@ static int iotlb_translate(const struct vringh *vrh,
} }
size = map->size - addr + map->start; size = map->size - addr + map->start;
pa = map->addr + addr - map->start; io_len = min(len - s, size);
pfn = pa >> PAGE_SHIFT; io_addr = map->addr - map->start + addr;
bvec_set_page(&iov[ret], pfn_to_page(pfn), min(len - s, size),
pa & (PAGE_SIZE - 1)); if (vrh->use_va) {
struct iovec *iovec = ivec->iov.iovec;
iovec[ret].iov_len = io_len;
iovec[ret].iov_base = (void __user *)io_addr;
} else {
u64 pfn = io_addr >> PAGE_SHIFT;
struct bio_vec *bvec = ivec->iov.bvec;
bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len,
io_addr & (PAGE_SIZE - 1));
}
s += size; s += size;
addr += size; addr += size;
++ret; ++ret;
...@@ -1141,26 +1162,41 @@ static int iotlb_translate(const struct vringh *vrh, ...@@ -1141,26 +1162,41 @@ static int iotlb_translate(const struct vringh *vrh,
return ret; return ret;
} }
#define IOTLB_IOV_STRIDE 16
static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
void *src, size_t len) void *src, size_t len)
{ {
struct iotlb_vec ivec;
union {
struct iovec iovec[IOTLB_IOV_STRIDE];
struct bio_vec bvec[IOTLB_IOV_STRIDE];
} iov;
u64 total_translated = 0; u64 total_translated = 0;
ivec.iov.iovec = iov.iovec;
ivec.count = IOTLB_IOV_STRIDE;
while (total_translated < len) { while (total_translated < len) {
struct bio_vec iov[16];
struct iov_iter iter; struct iov_iter iter;
u64 translated; u64 translated;
int ret; int ret;
ret = iotlb_translate(vrh, (u64)(uintptr_t)src, ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
len - total_translated, &translated, len - total_translated, &translated,
iov, ARRAY_SIZE(iov), VHOST_MAP_RO); &ivec, VHOST_MAP_RO);
if (ret == -ENOBUFS) if (ret == -ENOBUFS)
ret = ARRAY_SIZE(iov); ret = IOTLB_IOV_STRIDE;
else if (ret < 0) else if (ret < 0)
return ret; return ret;
iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated); if (vrh->use_va) {
iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret,
translated);
} else {
iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret,
translated);
}
ret = copy_from_iter(dst, translated, &iter); ret = copy_from_iter(dst, translated, &iter);
if (ret < 0) if (ret < 0)
...@@ -1177,23 +1213,36 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, ...@@ -1177,23 +1213,36 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
void *src, size_t len) void *src, size_t len)
{ {
struct iotlb_vec ivec;
union {
struct iovec iovec[IOTLB_IOV_STRIDE];
struct bio_vec bvec[IOTLB_IOV_STRIDE];
} iov;
u64 total_translated = 0; u64 total_translated = 0;
ivec.iov.iovec = iov.iovec;
ivec.count = IOTLB_IOV_STRIDE;
while (total_translated < len) { while (total_translated < len) {
struct bio_vec iov[16];
struct iov_iter iter; struct iov_iter iter;
u64 translated; u64 translated;
int ret; int ret;
ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
len - total_translated, &translated, len - total_translated, &translated,
iov, ARRAY_SIZE(iov), VHOST_MAP_WO); &ivec, VHOST_MAP_WO);
if (ret == -ENOBUFS) if (ret == -ENOBUFS)
ret = ARRAY_SIZE(iov); ret = IOTLB_IOV_STRIDE;
else if (ret < 0) else if (ret < 0)
return ret; return ret;
iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated); if (vrh->use_va) {
iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret,
translated);
} else {
iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret,
translated);
}
ret = copy_to_iter(src, translated, &iter); ret = copy_to_iter(src, translated, &iter);
if (ret < 0) if (ret < 0)
...@@ -1210,20 +1259,36 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, ...@@ -1210,20 +1259,36 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
static inline int getu16_iotlb(const struct vringh *vrh, static inline int getu16_iotlb(const struct vringh *vrh,
u16 *val, const __virtio16 *p) u16 *val, const __virtio16 *p)
{ {
struct bio_vec iov; struct iotlb_vec ivec;
void *kaddr, *from; union {
struct iovec iovec[1];
struct bio_vec bvec[1];
} iov;
__virtio16 tmp;
int ret; int ret;
ivec.iov.iovec = iov.iovec;
ivec.count = 1;
/* Atomic read is needed for getu16 */ /* Atomic read is needed for getu16 */
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
&iov, 1, VHOST_MAP_RO); NULL, &ivec, VHOST_MAP_RO);
if (ret < 0) if (ret < 0)
return ret; return ret;
kaddr = kmap_atomic(iov.bv_page); if (vrh->use_va) {
from = kaddr + iov.bv_offset; ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
*val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from)); if (ret)
kunmap_atomic(kaddr); return ret;
} else {
void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
void *from = kaddr + ivec.iov.bvec[0].bv_offset;
tmp = READ_ONCE(*(__virtio16 *)from);
kunmap_local(kaddr);
}
*val = vringh16_to_cpu(vrh, tmp);
return 0; return 0;
} }
...@@ -1231,20 +1296,36 @@ static inline int getu16_iotlb(const struct vringh *vrh, ...@@ -1231,20 +1296,36 @@ static inline int getu16_iotlb(const struct vringh *vrh,
static inline int putu16_iotlb(const struct vringh *vrh, static inline int putu16_iotlb(const struct vringh *vrh,
__virtio16 *p, u16 val) __virtio16 *p, u16 val)
{ {
struct bio_vec iov; struct iotlb_vec ivec;
void *kaddr, *to; union {
struct iovec iovec;
struct bio_vec bvec;
} iov;
__virtio16 tmp;
int ret; int ret;
ivec.iov.iovec = &iov.iovec;
ivec.count = 1;
/* Atomic write is needed for putu16 */ /* Atomic write is needed for putu16 */
ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
&iov, 1, VHOST_MAP_WO); NULL, &ivec, VHOST_MAP_RO);
if (ret < 0) if (ret < 0)
return ret; return ret;
kaddr = kmap_atomic(iov.bv_page); tmp = cpu_to_vringh16(vrh, val);
to = kaddr + iov.bv_offset;
WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val)); if (vrh->use_va) {
kunmap_atomic(kaddr); ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
if (ret)
return ret;
} else {
void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
void *to = kaddr + ivec.iov.bvec[0].bv_offset;
WRITE_ONCE(*(__virtio16 *)to, tmp);
kunmap_local(kaddr);
}
return 0; return 0;
} }
...@@ -1306,9 +1387,9 @@ static inline int putused_iotlb(const struct vringh *vrh, ...@@ -1306,9 +1387,9 @@ static inline int putused_iotlb(const struct vringh *vrh,
* @features: the feature bits for this ring. * @features: the feature bits for this ring.
* @num: the number of elements. * @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O. * @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userpace descriptor pointer. * @desc: the userspace descriptor pointer.
* @avail: the userpace avail pointer. * @avail: the userspace avail pointer.
* @used: the userpace used pointer. * @used: the userspace used pointer.
* *
* Returns an error if num is invalid. * Returns an error if num is invalid.
*/ */
...@@ -1318,11 +1399,39 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features, ...@@ -1318,11 +1399,39 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features,
struct vring_avail *avail, struct vring_avail *avail,
struct vring_used *used) struct vring_used *used)
{ {
vrh->use_va = false;
return vringh_init_kern(vrh, features, num, weak_barriers, return vringh_init_kern(vrh, features, num, weak_barriers,
desc, avail, used); desc, avail, used);
} }
EXPORT_SYMBOL(vringh_init_iotlb); EXPORT_SYMBOL(vringh_init_iotlb);
/**
* vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing
* user VA.
* @vrh: the vringh to initialize.
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
* @desc: the userspace descriptor pointer.
* @avail: the userspace avail pointer.
* @used: the userspace used pointer.
*
* Returns an error if num is invalid.
*/
int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
unsigned int num, bool weak_barriers,
struct vring_desc *desc,
struct vring_avail *avail,
struct vring_used *used)
{
vrh->use_va = true;
return vringh_init_kern(vrh, features, num, weak_barriers,
desc, avail, used);
}
EXPORT_SYMBOL(vringh_init_iotlb_va);
/** /**
* vringh_set_iotlb - initialize a vringh for a ring with IOTLB. * vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
* @vrh: the vring * @vrh: the vring
......
...@@ -286,6 +286,16 @@ static bool vm_notify(struct virtqueue *vq) ...@@ -286,6 +286,16 @@ static bool vm_notify(struct virtqueue *vq)
return true; return true;
} }
static bool vm_notify_with_data(struct virtqueue *vq)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
u32 data = vring_notification_data(vq);
writel(data, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
return true;
}
/* Notify all virtqueues on an interrupt. */ /* Notify all virtqueues on an interrupt. */
static irqreturn_t vm_interrupt(int irq, void *opaque) static irqreturn_t vm_interrupt(int irq, void *opaque)
{ {
...@@ -364,12 +374,18 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in ...@@ -364,12 +374,18 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
const char *name, bool ctx) const char *name, bool ctx)
{ {
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
bool (*notify)(struct virtqueue *vq);
struct virtio_mmio_vq_info *info; struct virtio_mmio_vq_info *info;
struct virtqueue *vq; struct virtqueue *vq;
unsigned long flags; unsigned long flags;
unsigned int num; unsigned int num;
int err; int err;
if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = vm_notify_with_data;
else
notify = vm_notify;
if (!name) if (!name)
return NULL; return NULL;
...@@ -398,7 +414,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in ...@@ -398,7 +414,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
/* Create the vring */ /* Create the vring */
vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev,
true, true, ctx, vm_notify, callback, name); true, true, ctx, notify, callback, name);
if (!vq) { if (!vq) {
err = -ENOMEM; err = -ENOMEM;
goto error_new_virtqueue; goto error_new_virtqueue;
......
...@@ -288,6 +288,15 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) ...@@ -288,6 +288,15 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
return vp_modern_config_vector(&vp_dev->mdev, vector); return vp_modern_config_vector(&vp_dev->mdev, vector);
} }
static bool vp_notify_with_data(struct virtqueue *vq)
{
u32 data = vring_notification_data(vq);
iowrite32(data, (void __iomem *)vq->priv);
return true;
}
static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
struct virtio_pci_vq_info *info, struct virtio_pci_vq_info *info,
unsigned int index, unsigned int index,
...@@ -298,10 +307,16 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, ...@@ -298,10 +307,16 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
{ {
struct virtio_pci_modern_device *mdev = &vp_dev->mdev; struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
bool (*notify)(struct virtqueue *vq);
struct virtqueue *vq; struct virtqueue *vq;
u16 num; u16 num;
int err; int err;
if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA))
notify = vp_notify_with_data;
else
notify = vp_notify;
if (index >= vp_modern_get_num_queues(mdev)) if (index >= vp_modern_get_num_queues(mdev))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
...@@ -310,18 +325,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, ...@@ -310,18 +325,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
if (!num || vp_modern_get_queue_enable(mdev, index)) if (!num || vp_modern_get_queue_enable(mdev, index))
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
if (!is_power_of_2(num)) {
dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);
return ERR_PTR(-EINVAL);
}
info->msix_vector = msix_vec; info->msix_vector = msix_vec;
/* create the vring */ /* create the vring */
vq = vring_create_virtqueue(index, num, vq = vring_create_virtqueue(index, num,
SMP_CACHE_BYTES, &vp_dev->vdev, SMP_CACHE_BYTES, &vp_dev->vdev,
true, true, ctx, true, true, ctx,
vp_notify, callback, name); notify, callback, name);
if (!vq) if (!vq)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
......
...@@ -231,9 +231,9 @@ static void vring_free(struct virtqueue *_vq); ...@@ -231,9 +231,9 @@ static void vring_free(struct virtqueue *_vq);
* Helpers. * Helpers.
*/ */
#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq, static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
unsigned int total_sg) unsigned int total_sg)
{ {
/* /*
...@@ -269,7 +269,7 @@ static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq, ...@@ -269,7 +269,7 @@ static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
* unconditionally on data path. * unconditionally on data path.
*/ */
static bool vring_use_dma_api(struct virtio_device *vdev) static bool vring_use_dma_api(const struct virtio_device *vdev)
{ {
if (!virtio_has_dma_quirk(vdev)) if (!virtio_has_dma_quirk(vdev))
return true; return true;
...@@ -289,7 +289,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev) ...@@ -289,7 +289,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev)
return false; return false;
} }
size_t virtio_max_dma_size(struct virtio_device *vdev) size_t virtio_max_dma_size(const struct virtio_device *vdev)
{ {
size_t max_segment_size = SIZE_MAX; size_t max_segment_size = SIZE_MAX;
...@@ -349,7 +349,7 @@ static void vring_free_queue(struct virtio_device *vdev, size_t size, ...@@ -349,7 +349,7 @@ static void vring_free_queue(struct virtio_device *vdev, size_t size,
* making all of the arch DMA ops work on the vring device itself * making all of the arch DMA ops work on the vring device itself
* is a mess. * is a mess.
*/ */
static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq) static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
{ {
return vq->dma_dev; return vq->dma_dev;
} }
...@@ -423,7 +423,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num) ...@@ -423,7 +423,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
*/ */
static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
struct vring_desc *desc) const struct vring_desc *desc)
{ {
u16 flags; u16 flags;
...@@ -784,7 +784,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, ...@@ -784,7 +784,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
} }
} }
static inline bool more_used_split(const struct vring_virtqueue *vq) static bool more_used_split(const struct vring_virtqueue *vq)
{ {
return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
vq->split.vring.used->idx); vq->split.vring.used->idx);
...@@ -854,6 +854,14 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq) ...@@ -854,6 +854,14 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
if (vq->event) if (vq->event)
/* TODO: this is a hack. Figure out a cleaner value to write. */ /* TODO: this is a hack. Figure out a cleaner value to write. */
vring_used_event(&vq->split.vring) = 0x0; vring_used_event(&vq->split.vring) = 0x0;
...@@ -1172,18 +1180,18 @@ static int virtqueue_resize_split(struct virtqueue *_vq, u32 num) ...@@ -1172,18 +1180,18 @@ static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
/* /*
* Packed ring specific functions - *_packed(). * Packed ring specific functions - *_packed().
*/ */
static inline bool packed_used_wrap_counter(u16 last_used_idx) static bool packed_used_wrap_counter(u16 last_used_idx)
{ {
return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR)); return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
} }
static inline u16 packed_last_used(u16 last_used_idx) static u16 packed_last_used(u16 last_used_idx)
{ {
return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR)); return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
} }
static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
struct vring_desc_extra *extra) const struct vring_desc_extra *extra)
{ {
u16 flags; u16 flags;
...@@ -1206,7 +1214,7 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, ...@@ -1206,7 +1214,7 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
} }
static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
struct vring_packed_desc *desc) const struct vring_packed_desc *desc)
{ {
u16 flags; u16 flags;
...@@ -1612,7 +1620,7 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq, ...@@ -1612,7 +1620,7 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
return avail == used && used == used_wrap_counter; return avail == used && used == used_wrap_counter;
} }
static inline bool more_used_packed(const struct vring_virtqueue *vq) static bool more_used_packed(const struct vring_virtqueue *vq)
{ {
u16 last_used; u16 last_used;
u16 last_used_idx; u16 last_used_idx;
...@@ -1699,6 +1707,14 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq) ...@@ -1699,6 +1707,14 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
vq->packed.vring.driver->flags = vq->packed.vring.driver->flags =
cpu_to_le16(vq->packed.event_flags_shadow); cpu_to_le16(vq->packed.event_flags_shadow);
} }
...@@ -2330,12 +2346,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq) ...@@ -2330,12 +2346,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); struct vring_virtqueue *vq = to_vvq(_vq);
/* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
if (vq->packed_ring) if (vq->packed_ring)
virtqueue_disable_cb_packed(_vq); virtqueue_disable_cb_packed(_vq);
else else
...@@ -2752,6 +2762,23 @@ void vring_del_virtqueue(struct virtqueue *_vq) ...@@ -2752,6 +2762,23 @@ void vring_del_virtqueue(struct virtqueue *_vq)
} }
EXPORT_SYMBOL_GPL(vring_del_virtqueue); EXPORT_SYMBOL_GPL(vring_del_virtqueue);
u32 vring_notification_data(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 next;
if (vq->packed_ring)
next = (vq->packed.next_avail_idx &
~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
vq->packed.avail_wrap_counter <<
VRING_PACKED_EVENT_F_WRAP_CTR;
else
next = vq->split.avail_idx_shadow;
return next << 16 | _vq->index;
}
EXPORT_SYMBOL_GPL(vring_notification_data);
/* Manipulates transport-specific feature bits. */ /* Manipulates transport-specific feature bits. */
void vring_transport_features(struct virtio_device *vdev) void vring_transport_features(struct virtio_device *vdev)
{ {
...@@ -2771,6 +2798,8 @@ void vring_transport_features(struct virtio_device *vdev) ...@@ -2771,6 +2798,8 @@ void vring_transport_features(struct virtio_device *vdev)
break; break;
case VIRTIO_F_ORDER_PLATFORM: case VIRTIO_F_ORDER_PLATFORM:
break; break;
case VIRTIO_F_NOTIFICATION_DATA:
break;
default: default:
/* We don't understand this bit. */ /* We don't understand this bit. */
__virtio_clear_bit(vdev, i); __virtio_clear_bit(vdev, i);
...@@ -2786,10 +2815,10 @@ EXPORT_SYMBOL_GPL(vring_transport_features); ...@@ -2786,10 +2815,10 @@ EXPORT_SYMBOL_GPL(vring_transport_features);
* Returns the size of the vring. This is mainly used for boasting to * Returns the size of the vring. This is mainly used for boasting to
* userspace. Unlike other operations, this need not be serialized. * userspace. Unlike other operations, this need not be serialized.
*/ */
unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); const struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
} }
...@@ -2819,9 +2848,9 @@ void __virtqueue_unbreak(struct virtqueue *_vq) ...@@ -2819,9 +2848,9 @@ void __virtqueue_unbreak(struct virtqueue *_vq)
} }
EXPORT_SYMBOL_GPL(__virtqueue_unbreak); EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
bool virtqueue_is_broken(struct virtqueue *_vq) bool virtqueue_is_broken(const struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); const struct vring_virtqueue *vq = to_vvq(_vq);
return READ_ONCE(vq->broken); return READ_ONCE(vq->broken);
} }
...@@ -2868,9 +2897,9 @@ void __virtio_unbreak_device(struct virtio_device *dev) ...@@ -2868,9 +2897,9 @@ void __virtio_unbreak_device(struct virtio_device *dev)
} }
EXPORT_SYMBOL_GPL(__virtio_unbreak_device); EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring); BUG_ON(!vq->we_own_ring);
...@@ -2881,9 +2910,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) ...@@ -2881,9 +2910,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
} }
EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr); EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring); BUG_ON(!vq->we_own_ring);
...@@ -2895,9 +2924,9 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) ...@@ -2895,9 +2924,9 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
} }
EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr); EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
{ {
struct vring_virtqueue *vq = to_vvq(_vq); const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring); BUG_ON(!vq->we_own_ring);
...@@ -2910,7 +2939,7 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) ...@@ -2910,7 +2939,7 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
/* Only available for split ring */ /* Only available for split ring */
const struct vring *virtqueue_get_vring(struct virtqueue *vq) const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
{ {
return &to_vvq(vq)->split.vring; return &to_vvq(vq)->split.vring;
} }
......
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/uuid.h> #include <linux/uuid.h>
#include <linux/group_cpus.h>
#include <linux/virtio.h> #include <linux/virtio.h>
#include <linux/vdpa.h> #include <linux/vdpa.h>
#include <linux/virtio_config.h> #include <linux/virtio_config.h>
...@@ -112,6 +113,17 @@ static bool virtio_vdpa_notify(struct virtqueue *vq) ...@@ -112,6 +113,17 @@ static bool virtio_vdpa_notify(struct virtqueue *vq)
return true; return true;
} }
static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
{
struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
const struct vdpa_config_ops *ops = vdpa->config;
u32 data = vring_notification_data(vq);
ops->kick_vq_with_data(vdpa, data);
return true;
}
static irqreturn_t virtio_vdpa_config_cb(void *private) static irqreturn_t virtio_vdpa_config_cb(void *private)
{ {
struct virtio_vdpa_device *vd_dev = private; struct virtio_vdpa_device *vd_dev = private;
...@@ -138,6 +150,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, ...@@ -138,6 +150,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
struct device *dma_dev; struct device *dma_dev;
const struct vdpa_config_ops *ops = vdpa->config; const struct vdpa_config_ops *ops = vdpa->config;
struct virtio_vdpa_vq_info *info; struct virtio_vdpa_vq_info *info;
bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
struct vdpa_callback cb; struct vdpa_callback cb;
struct virtqueue *vq; struct virtqueue *vq;
u64 desc_addr, driver_addr, device_addr; u64 desc_addr, driver_addr, device_addr;
...@@ -154,6 +167,14 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, ...@@ -154,6 +167,14 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
if (index >= vdpa->nvqs) if (index >= vdpa->nvqs)
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
/* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
if (ops->kick_vq_with_data)
notify = virtio_vdpa_notify_with_data;
else
__virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
}
/* Queue shouldn't already be set up. */ /* Queue shouldn't already be set up. */
if (ops->get_vq_ready(vdpa, index)) if (ops->get_vq_ready(vdpa, index))
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
...@@ -183,8 +204,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, ...@@ -183,8 +204,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
dma_dev = vdpa_get_dma_dev(vdpa); dma_dev = vdpa_get_dma_dev(vdpa);
vq = vring_create_virtqueue_dma(index, max_num, align, vdev, vq = vring_create_virtqueue_dma(index, max_num, align, vdev,
true, may_reduce_num, ctx, true, may_reduce_num, ctx,
virtio_vdpa_notify, callback, notify, callback, name, dma_dev);
name, dma_dev);
if (!vq) { if (!vq) {
err = -ENOMEM; err = -ENOMEM;
goto error_new_virtqueue; goto error_new_virtqueue;
...@@ -195,6 +215,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, ...@@ -195,6 +215,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
/* Setup virtqueue callback */ /* Setup virtqueue callback */
cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
cb.private = info; cb.private = info;
cb.trigger = NULL;
ops->set_vq_cb(vdpa, index, &cb); ops->set_vq_cb(vdpa, index, &cb);
ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
...@@ -272,6 +293,66 @@ static void virtio_vdpa_del_vqs(struct virtio_device *vdev) ...@@ -272,6 +293,66 @@ static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
virtio_vdpa_del_vq(vq); virtio_vdpa_del_vq(vq);
} }
static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
{
affd->nr_sets = 1;
affd->set_size[0] = affvecs;
}
static struct cpumask *
create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
{
unsigned int affvecs = 0, curvec, usedvecs, i;
struct cpumask *masks = NULL;
if (nvecs > affd->pre_vectors + affd->post_vectors)
affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
if (!affd->calc_sets)
affd->calc_sets = default_calc_sets;
affd->calc_sets(affd, affvecs);
if (!affvecs)
return NULL;
masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
if (!masks)
return NULL;
/* Fill out vectors at the beginning that don't need affinity */
for (curvec = 0; curvec < affd->pre_vectors; curvec++)
cpumask_setall(&masks[curvec]);
for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
unsigned int this_vecs = affd->set_size[i];
int j;
struct cpumask *result = group_cpus_evenly(this_vecs);
if (!result) {
kfree(masks);
return NULL;
}
for (j = 0; j < this_vecs; j++)
cpumask_copy(&masks[curvec + j], &result[j]);
kfree(result);
curvec += this_vecs;
usedvecs += this_vecs;
}
/* Fill out vectors at the end that don't need affinity */
if (usedvecs >= affvecs)
curvec = affd->pre_vectors + affvecs;
else
curvec = affd->pre_vectors + usedvecs;
for (; curvec < nvecs; curvec++)
cpumask_setall(&masks[curvec]);
return masks;
}
static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[], struct virtqueue *vqs[],
vq_callback_t *callbacks[], vq_callback_t *callbacks[],
...@@ -282,9 +363,15 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, ...@@ -282,9 +363,15 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
struct vdpa_device *vdpa = vd_get_vdpa(vdev); struct vdpa_device *vdpa = vd_get_vdpa(vdev);
const struct vdpa_config_ops *ops = vdpa->config; const struct vdpa_config_ops *ops = vdpa->config;
struct irq_affinity default_affd = { 0 };
struct cpumask *masks;
struct vdpa_callback cb; struct vdpa_callback cb;
int i, err, queue_idx = 0; int i, err, queue_idx = 0;
masks = create_affinity_masks(nvqs, desc ? desc : &default_affd);
if (!masks)
return -ENOMEM;
for (i = 0; i < nvqs; ++i) { for (i = 0; i < nvqs; ++i) {
if (!names[i]) { if (!names[i]) {
vqs[i] = NULL; vqs[i] = NULL;
...@@ -298,6 +385,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, ...@@ -298,6 +385,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
err = PTR_ERR(vqs[i]); err = PTR_ERR(vqs[i]);
goto err_setup_vq; goto err_setup_vq;
} }
ops->set_vq_affinity(vdpa, i, &masks[i]);
} }
cb.callback = virtio_vdpa_config_cb; cb.callback = virtio_vdpa_config_cb;
...@@ -337,6 +425,32 @@ static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) ...@@ -337,6 +425,32 @@ static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
return dev_name(&vdpa->dev); return dev_name(&vdpa->dev);
} }
static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
const struct cpumask *cpu_mask)
{
struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
struct vdpa_device *vdpa = vd_dev->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
unsigned int index = vq->index;
if (ops->set_vq_affinity)
return ops->set_vq_affinity(vdpa, index, cpu_mask);
return 0;
}
static const struct cpumask *
virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
{
struct vdpa_device *vdpa = vd_get_vdpa(vdev);
const struct vdpa_config_ops *ops = vdpa->config;
if (ops->get_vq_affinity)
return ops->get_vq_affinity(vdpa, index);
return NULL;
}
static const struct virtio_config_ops virtio_vdpa_config_ops = { static const struct virtio_config_ops virtio_vdpa_config_ops = {
.get = virtio_vdpa_get, .get = virtio_vdpa_get,
.set = virtio_vdpa_set, .set = virtio_vdpa_set,
...@@ -349,6 +463,8 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = { ...@@ -349,6 +463,8 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = {
.get_features = virtio_vdpa_get_features, .get_features = virtio_vdpa_get_features,
.finalize_features = virtio_vdpa_finalize_features, .finalize_features = virtio_vdpa_finalize_features,
.bus_name = virtio_vdpa_bus_name, .bus_name = virtio_vdpa_bus_name,
.set_vq_affinity = virtio_vdpa_set_vq_affinity,
.get_vq_affinity = virtio_vdpa_get_vq_affinity,
}; };
static void virtio_vdpa_release_dev(struct device *_d) static void virtio_vdpa_release_dev(struct device *_d)
......
...@@ -10,13 +10,19 @@ ...@@ -10,13 +10,19 @@
#include <linux/if_ether.h> #include <linux/if_ether.h>
/** /**
* struct vdpa_calllback - vDPA callback definition. * struct vdpa_callback - vDPA callback definition.
* @callback: interrupt callback function * @callback: interrupt callback function
* @private: the data passed to the callback function * @private: the data passed to the callback function
* @trigger: the eventfd for the callback (Optional).
* When it is set, the vDPA driver must guarantee that
* signaling it is functional equivalent to triggering
* the callback. Then vDPA parent can signal it directly
* instead of triggering the callback.
*/ */
struct vdpa_callback { struct vdpa_callback {
irqreturn_t (*callback)(void *data); irqreturn_t (*callback)(void *data);
void *private; void *private;
struct eventfd_ctx *trigger;
}; };
/** /**
...@@ -114,7 +120,7 @@ struct vdpa_dev_set_config { ...@@ -114,7 +120,7 @@ struct vdpa_dev_set_config {
}; };
/** /**
* Corresponding file area for device memory mapping * struct vdpa_map_file - file area for device memory mapping
* @file: vma->vm_file for the mapping * @file: vma->vm_file for the mapping
* @offset: mapping offset in the vm_file * @offset: mapping offset in the vm_file
*/ */
...@@ -143,6 +149,14 @@ struct vdpa_map_file { ...@@ -143,6 +149,14 @@ struct vdpa_map_file {
* @kick_vq: Kick the virtqueue * @kick_vq: Kick the virtqueue
* @vdev: vdpa device * @vdev: vdpa device
* @idx: virtqueue index * @idx: virtqueue index
* @kick_vq_with_data: Kick the virtqueue and supply extra data
* (only if VIRTIO_F_NOTIFICATION_DATA is negotiated)
* @vdev: vdpa device
* @data for split virtqueue:
* 16 bits vqn and 16 bits next available index.
* @data for packed virtqueue:
* 16 bits vqn, 15 least significant bits of
* next available index and 1 bit next_wrap.
* @set_vq_cb: Set the interrupt callback function for * @set_vq_cb: Set the interrupt callback function for
* a virtqueue * a virtqueue
* @vdev: vdpa device * @vdev: vdpa device
...@@ -165,10 +179,16 @@ struct vdpa_map_file { ...@@ -165,10 +179,16 @@ struct vdpa_map_file {
* @vdev: vdpa device * @vdev: vdpa device
* @idx: virtqueue index * @idx: virtqueue index
* @state: pointer to returned state (last_avail_idx) * @state: pointer to returned state (last_avail_idx)
* @get_vendor_vq_stats: Get the vendor statistics of a device.
* @vdev: vdpa device
* @idx: virtqueue index
* @msg: socket buffer holding stats message
* @extack: extack for reporting error messages
* Returns integer: success (0) or error (< 0)
* @get_vq_notification: Get the notification area for a virtqueue (optional) * @get_vq_notification: Get the notification area for a virtqueue (optional)
* @vdev: vdpa device * @vdev: vdpa device
* @idx: virtqueue index * @idx: virtqueue index
* Returns the notifcation area * Returns the notification area
* @get_vq_irq: Get the irq number of a virtqueue (optional, * @get_vq_irq: Get the irq number of a virtqueue (optional,
* but must implemented if require vq irq offloading) * but must implemented if require vq irq offloading)
* @vdev: vdpa device * @vdev: vdpa device
...@@ -250,6 +270,15 @@ struct vdpa_map_file { ...@@ -250,6 +270,15 @@ struct vdpa_map_file {
* @vdev: vdpa device * @vdev: vdpa device
* Returns the iova range supported by * Returns the iova range supported by
* the device. * the device.
* @set_vq_affinity: Set the affinity of virtqueue (optional)
* @vdev: vdpa device
* @idx: virtqueue index
* @cpu_mask: the affinity mask
* Returns integer: success (0) or error (< 0)
* @get_vq_affinity: Get the affinity of virtqueue (optional)
* @vdev: vdpa device
* @idx: virtqueue index
* Returns the affinity mask
* @set_group_asid: Set address space identifier for a * @set_group_asid: Set address space identifier for a
* virtqueue group (optional) * virtqueue group (optional)
* @vdev: vdpa device * @vdev: vdpa device
...@@ -290,6 +319,14 @@ struct vdpa_map_file { ...@@ -290,6 +319,14 @@ struct vdpa_map_file {
* @vdev: vdpa device * @vdev: vdpa device
* @idx: virtqueue index * @idx: virtqueue index
* Returns pointer to structure device or error (NULL) * Returns pointer to structure device or error (NULL)
* @bind_mm: Bind the device to a specific address space
* so the vDPA framework can use VA when this
* callback is implemented. (optional)
* @vdev: vdpa device
* @mm: address space to bind
* @unbind_mm: Unbind the device from the address space
* bound using the bind_mm callback. (optional)
* @vdev: vdpa device
* @free: Free resources that belongs to vDPA (optional) * @free: Free resources that belongs to vDPA (optional)
* @vdev: vdpa device * @vdev: vdpa device
*/ */
...@@ -300,6 +337,7 @@ struct vdpa_config_ops { ...@@ -300,6 +337,7 @@ struct vdpa_config_ops {
u64 device_area); u64 device_area);
void (*set_vq_num)(struct vdpa_device *vdev, u16 idx, u32 num); void (*set_vq_num)(struct vdpa_device *vdev, u16 idx, u32 num);
void (*kick_vq)(struct vdpa_device *vdev, u16 idx); void (*kick_vq)(struct vdpa_device *vdev, u16 idx);
void (*kick_vq_with_data)(struct vdpa_device *vdev, u32 data);
void (*set_vq_cb)(struct vdpa_device *vdev, u16 idx, void (*set_vq_cb)(struct vdpa_device *vdev, u16 idx,
struct vdpa_callback *cb); struct vdpa_callback *cb);
void (*set_vq_ready)(struct vdpa_device *vdev, u16 idx, bool ready); void (*set_vq_ready)(struct vdpa_device *vdev, u16 idx, bool ready);
...@@ -340,6 +378,10 @@ struct vdpa_config_ops { ...@@ -340,6 +378,10 @@ struct vdpa_config_ops {
const void *buf, unsigned int len); const void *buf, unsigned int len);
u32 (*get_generation)(struct vdpa_device *vdev); u32 (*get_generation)(struct vdpa_device *vdev);
struct vdpa_iova_range (*get_iova_range)(struct vdpa_device *vdev); struct vdpa_iova_range (*get_iova_range)(struct vdpa_device *vdev);
int (*set_vq_affinity)(struct vdpa_device *vdev, u16 idx,
const struct cpumask *cpu_mask);
const struct cpumask *(*get_vq_affinity)(struct vdpa_device *vdev,
u16 idx);
/* DMA ops */ /* DMA ops */
int (*set_map)(struct vdpa_device *vdev, unsigned int asid, int (*set_map)(struct vdpa_device *vdev, unsigned int asid,
...@@ -351,6 +393,8 @@ struct vdpa_config_ops { ...@@ -351,6 +393,8 @@ struct vdpa_config_ops {
int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group, int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group,
unsigned int asid); unsigned int asid);
struct device *(*get_vq_dma_dev)(struct vdpa_device *vdev, u16 idx); struct device *(*get_vq_dma_dev)(struct vdpa_device *vdev, u16 idx);
int (*bind_mm)(struct vdpa_device *vdev, struct mm_struct *mm);
void (*unbind_mm)(struct vdpa_device *vdev);
/* Free device resources */ /* Free device resources */
void (*free)(struct vdpa_device *vdev); void (*free)(struct vdpa_device *vdev);
...@@ -506,6 +550,8 @@ struct vdpa_mgmtdev_ops { ...@@ -506,6 +550,8 @@ struct vdpa_mgmtdev_ops {
* @config_attr_mask: bit mask of attributes of type enum vdpa_attr that * @config_attr_mask: bit mask of attributes of type enum vdpa_attr that
* management device support during dev_add callback * management device support during dev_add callback
* @list: list entry * @list: list entry
* @supported_features: features supported by device
* @max_supported_vqs: maximum number of virtqueues supported by device
*/ */
struct vdpa_mgmt_dev { struct vdpa_mgmt_dev {
struct device *device; struct device *device;
......
...@@ -34,8 +34,8 @@ struct virtqueue { ...@@ -34,8 +34,8 @@ struct virtqueue {
unsigned int index; unsigned int index;
unsigned int num_free; unsigned int num_free;
unsigned int num_max; unsigned int num_max;
void *priv;
bool reset; bool reset;
void *priv;
}; };
int virtqueue_add_outbuf(struct virtqueue *vq, int virtqueue_add_outbuf(struct virtqueue *vq,
...@@ -84,14 +84,14 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *vq); ...@@ -84,14 +84,14 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *vq);
void *virtqueue_detach_unused_buf(struct virtqueue *vq); void *virtqueue_detach_unused_buf(struct virtqueue *vq);
unsigned int virtqueue_get_vring_size(struct virtqueue *vq); unsigned int virtqueue_get_vring_size(const struct virtqueue *vq);
bool virtqueue_is_broken(struct virtqueue *vq); bool virtqueue_is_broken(const struct virtqueue *vq);
const struct vring *virtqueue_get_vring(struct virtqueue *vq); const struct vring *virtqueue_get_vring(const struct virtqueue *vq);
dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *vq);
dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *vq);
dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_used_addr(const struct virtqueue *vq);
int virtqueue_resize(struct virtqueue *vq, u32 num, int virtqueue_resize(struct virtqueue *vq, u32 num,
void (*recycle)(struct virtqueue *vq, void *buf)); void (*recycle)(struct virtqueue *vq, void *buf));
...@@ -147,7 +147,7 @@ int virtio_device_restore(struct virtio_device *dev); ...@@ -147,7 +147,7 @@ int virtio_device_restore(struct virtio_device *dev);
#endif #endif
void virtio_reset_device(struct virtio_device *dev); void virtio_reset_device(struct virtio_device *dev);
size_t virtio_max_dma_size(struct virtio_device *vdev); size_t virtio_max_dma_size(const struct virtio_device *vdev);
#define virtio_device_for_each_vq(vdev, vq) \ #define virtio_device_for_each_vq(vdev, vq) \
list_for_each_entry(vq, &vdev->vqs, list) list_for_each_entry(vq, &vdev->vqs, list)
......
...@@ -58,6 +58,7 @@ do { \ ...@@ -58,6 +58,7 @@ do { \
struct virtio_device; struct virtio_device;
struct virtqueue; struct virtqueue;
struct device;
/* /*
* Creates a virtqueue and allocates the descriptor ring. If * Creates a virtqueue and allocates the descriptor ring. If
...@@ -117,4 +118,6 @@ void vring_del_virtqueue(struct virtqueue *vq); ...@@ -117,4 +118,6 @@ void vring_del_virtqueue(struct virtqueue *vq);
void vring_transport_features(struct virtio_device *vdev); void vring_transport_features(struct virtio_device *vdev);
irqreturn_t vring_interrupt(int irq, void *_vq); irqreturn_t vring_interrupt(int irq, void *_vq);
u32 vring_notification_data(struct virtqueue *_vq);
#endif /* _LINUX_VIRTIO_RING_H */ #endif /* _LINUX_VIRTIO_RING_H */
...@@ -32,6 +32,9 @@ struct vringh { ...@@ -32,6 +32,9 @@ struct vringh {
/* Can we get away with weak barriers? */ /* Can we get away with weak barriers? */
bool weak_barriers; bool weak_barriers;
/* Use user's VA */
bool use_va;
/* Last available index we saw (ie. where we're up to). */ /* Last available index we saw (ie. where we're up to). */
u16 last_avail_idx; u16 last_avail_idx;
...@@ -54,6 +57,9 @@ struct vringh { ...@@ -54,6 +57,9 @@ struct vringh {
void (*notify)(struct vringh *); void (*notify)(struct vringh *);
}; };
struct virtio_device;
typedef void vrh_callback_t(struct virtio_device *, struct vringh *);
/** /**
* struct vringh_config_ops - ops for creating a host vring from a virtio driver * struct vringh_config_ops - ops for creating a host vring from a virtio driver
* @find_vrhs: find the host vrings and instantiate them * @find_vrhs: find the host vrings and instantiate them
...@@ -65,8 +71,6 @@ struct vringh { ...@@ -65,8 +71,6 @@ struct vringh {
* Returns 0 on success or error status * Returns 0 on success or error status
* @del_vrhs: free the host vrings found by find_vrhs(). * @del_vrhs: free the host vrings found by find_vrhs().
*/ */
struct virtio_device;
typedef void vrh_callback_t(struct virtio_device *, struct vringh *);
struct vringh_config_ops { struct vringh_config_ops {
int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs, int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs,
struct vringh *vrhs[], vrh_callback_t *callbacks[]); struct vringh *vrhs[], vrh_callback_t *callbacks[]);
...@@ -81,6 +85,12 @@ struct vringh_range { ...@@ -81,6 +85,12 @@ struct vringh_range {
/** /**
* struct vringh_iov - iovec mangler. * struct vringh_iov - iovec mangler.
* @iov: array of iovecs to operate on
* @consumed: number of bytes consumed within iov[i]
* @i: index of current iovec
* @used: number of iovecs present in @iov
* @max_num: maximum number of iovecs.
* corresponds to allocated memory of @iov
* *
* Mangles iovec in place, and restores it. * Mangles iovec in place, and restores it.
* Remaining data is iov + i, of used - i elements. * Remaining data is iov + i, of used - i elements.
...@@ -93,6 +103,12 @@ struct vringh_iov { ...@@ -93,6 +103,12 @@ struct vringh_iov {
/** /**
* struct vringh_kiov - kvec mangler. * struct vringh_kiov - kvec mangler.
* @iov: array of iovecs to operate on
* @consumed: number of bytes consumed within iov[i]
* @i: index of current iovec
* @used: number of iovecs present in @iov
* @max_num: maximum number of iovecs.
* corresponds to allocated memory of @iov
* *
* Mangles kvec in place, and restores it. * Mangles kvec in place, and restores it.
* Remaining data is iov + i, of used - i elements. * Remaining data is iov + i, of used - i elements.
...@@ -284,6 +300,12 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features, ...@@ -284,6 +300,12 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features,
struct vring_avail *avail, struct vring_avail *avail,
struct vring_used *used); struct vring_used *used);
int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
unsigned int num, bool weak_barriers,
struct vring_desc *desc,
struct vring_avail *avail,
struct vring_used *used);
int vringh_getdesc_iotlb(struct vringh *vrh, int vringh_getdesc_iotlb(struct vringh *vrh,
struct vringh_kiov *riov, struct vringh_kiov *riov,
struct vringh_kiov *wiov, struct vringh_kiov *wiov,
......
...@@ -99,6 +99,12 @@ ...@@ -99,6 +99,12 @@
*/ */
#define VIRTIO_F_SR_IOV 37 #define VIRTIO_F_SR_IOV 37
/*
* This feature indicates that the driver passes extra data (besides
* identifying the virtqueue) in its device notifications.
*/
#define VIRTIO_F_NOTIFICATION_DATA 38
/* /*
* This feature indicates that the driver can reset a queue individually. * This feature indicates that the driver can reset a queue individually.
*/ */
......
...@@ -426,3 +426,4 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) ...@@ -426,3 +426,4 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps)
return masks; return masks;
} }
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
EXPORT_SYMBOL_GPL(group_cpus_evenly);
...@@ -49,7 +49,12 @@ typedef __s8 s8; ...@@ -49,7 +49,12 @@ typedef __s8 s8;
#endif #endif
#define __force #define __force
/* This is defined in linux/compiler_types.h and is left for backward
* compatibility.
*/
#ifndef __user
#define __user #define __user
#endif
#define __must_check #define __must_check
#define __cold #define __cold
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
#ifndef LINUX_COMPILER_H #ifndef LINUX_COMPILER_H
#define LINUX_COMPILER_H #define LINUX_COMPILER_H
#include "../../../include/linux/compiler_types.h"
#define WRITE_ONCE(var, val) \ #define WRITE_ONCE(var, val) \
(*((volatile typeof(val) *)(&(var))) = (val)) (*((volatile typeof(val) *)(&(var))) = (val))
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <stdarg.h> #include <stdarg.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include "../../../include/linux/container_of.h"
#include <linux/log2.h> #include <linux/log2.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/overflow.h> #include <linux/overflow.h>
...@@ -107,10 +108,6 @@ static inline void free_page(unsigned long addr) ...@@ -107,10 +108,6 @@ static inline void free_page(unsigned long addr)
free((void *)addr); free((void *)addr);
} }
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
# ifndef likely # ifndef likely
# define likely(x) (__builtin_expect(!!(x), 1)) # define likely(x) (__builtin_expect(!!(x), 1))
# endif # endif
......
...@@ -6,15 +6,10 @@ ...@@ -6,15 +6,10 @@
extern void *__user_addr_min, *__user_addr_max; extern void *__user_addr_min, *__user_addr_max;
static inline void __chk_user_ptr(const volatile void *p, size_t size)
{
assert(p >= __user_addr_min && p + size <= __user_addr_max);
}
#define put_user(x, ptr) \ #define put_user(x, ptr) \
({ \ ({ \
typeof(ptr) __pu_ptr = (ptr); \ typeof(ptr) __pu_ptr = (ptr); \
__chk_user_ptr(__pu_ptr, sizeof(*__pu_ptr)); \ __chk_user_ptr(__pu_ptr); \
WRITE_ONCE(*(__pu_ptr), x); \ WRITE_ONCE(*(__pu_ptr), x); \
0; \ 0; \
}) })
...@@ -22,7 +17,7 @@ static inline void __chk_user_ptr(const volatile void *p, size_t size) ...@@ -22,7 +17,7 @@ static inline void __chk_user_ptr(const volatile void *p, size_t size)
#define get_user(x, ptr) \ #define get_user(x, ptr) \
({ \ ({ \
typeof(ptr) __pu_ptr = (ptr); \ typeof(ptr) __pu_ptr = (ptr); \
__chk_user_ptr(__pu_ptr, sizeof(*__pu_ptr)); \ __chk_user_ptr(__pu_ptr); \
x = READ_ONCE(*(__pu_ptr)); \ x = READ_ONCE(*(__pu_ptr)); \
0; \ 0; \
}) })
...@@ -37,7 +32,6 @@ static void volatile_memcpy(volatile char *to, const volatile char *from, ...@@ -37,7 +32,6 @@ static void volatile_memcpy(volatile char *to, const volatile char *from,
static inline int copy_from_user(void *to, const void __user volatile *from, static inline int copy_from_user(void *to, const void __user volatile *from,
unsigned long n) unsigned long n)
{ {
__chk_user_ptr(from, n);
volatile_memcpy(to, from, n); volatile_memcpy(to, from, n);
return 0; return 0;
} }
...@@ -45,7 +39,6 @@ static inline int copy_from_user(void *to, const void __user volatile *from, ...@@ -45,7 +39,6 @@ static inline int copy_from_user(void *to, const void __user volatile *from,
static inline int copy_to_user(void __user volatile *to, const void *from, static inline int copy_to_user(void __user volatile *to, const void *from,
unsigned long n) unsigned long n)
{ {
__chk_user_ptr(to, n);
volatile_memcpy(to, from, n); volatile_memcpy(to, from, n);
return 0; return 0;
} }
......
...@@ -327,7 +327,7 @@ const struct option longopts[] = { ...@@ -327,7 +327,7 @@ const struct option longopts[] = {
} }
}; };
static void help(void) static void help(int status)
{ {
fprintf(stderr, "Usage: virtio_test [--help]" fprintf(stderr, "Usage: virtio_test [--help]"
" [--no-indirect]" " [--no-indirect]"
...@@ -337,6 +337,8 @@ static void help(void) ...@@ -337,6 +337,8 @@ static void help(void)
" [--batch=random/N]" " [--batch=random/N]"
" [--reset=N]" " [--reset=N]"
"\n"); "\n");
exit(status);
} }
int main(int argc, char **argv) int main(int argc, char **argv)
...@@ -354,14 +356,12 @@ int main(int argc, char **argv) ...@@ -354,14 +356,12 @@ int main(int argc, char **argv)
case -1: case -1:
goto done; goto done;
case '?': case '?':
help(); help(2);
exit(2);
case 'e': case 'e':
features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX); features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX);
break; break;
case 'h': case 'h':
help(); help(0);
goto done;
case 'i': case 'i':
features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC); features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC);
break; break;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment