Re: [Qemu-devel] [PATCH 7/9] virtio-scsi: use aio handler for data plane

[Fam Zheng]

On Fri, 04/01 15:19, Paolo Bonzini wrote:
> In addition to handling IO in vcpu thread and in io thread, dataplane
> introduces yet another mode: handling it by aio.
> 
> This reuses the same handler as previous modes, which triggers races as
> these were not designed to be reentrant.
> 
> Use a separate handler just for aio, and disable regular handlers when
> dataplane is active.
> 
> Reviewed-by: Cornelia Huck <address@hidden>
> Signed-off-by: Paolo Bonzini <address@hidden>
> ---
>  hw/scsi/virtio-scsi-dataplane.c | 43 ++++++++++++++++++++++++---
>  hw/scsi/virtio-scsi.c           | 65 
> ++++++++++++++++++++++++++++-------------
>  include/hw/virtio/virtio-scsi.h |  6 ++--
>  3 files changed, 86 insertions(+), 28 deletions(-)
> 
> diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c
> index 21d5bfd..a497a2c 100644
> --- a/hw/scsi/virtio-scsi-dataplane.c
> +++ b/hw/scsi/virtio-scsi-dataplane.c
> @@ -38,7 +38,35 @@ void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread 
> *iothread)
>      }
>  }
>  
> -static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue *vq, int n)
> +static void virtio_scsi_data_plane_handle_cmd(VirtIODevice *vdev,
> +                                              VirtQueue *vq)
> +{
> +    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
> +
> +    assert(s->ctx && s->dataplane_started);
> +    virtio_scsi_handle_cmd_vq(s, vq);
> +}
> +
> +static void virtio_scsi_data_plane_handle_ctrl(VirtIODevice *vdev,
> +                                               VirtQueue *vq)
> +{
> +    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
> +
> +    assert(s->ctx && s->dataplane_started);
> +    virtio_scsi_handle_ctrl_vq(s, vq);
> +}
> +
> +static void virtio_scsi_data_plane_handle_event(VirtIODevice *vdev,
> +                                                VirtQueue *vq)
> +{
> +    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
> +
> +    assert(s->ctx && s->dataplane_started);
> +    virtio_scsi_handle_event_vq(s, vq);
> +}
> +
> +static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue *vq, int n,
> +                               void (*fn)(VirtIODevice *vdev, VirtQueue *vq))

Are tabs used by mistake? Three more occasions below...

>  {
>      BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
>      VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
> @@ -54,6 +82,7 @@ static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue 
> *vq, int n)
>      }
>  
>      virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, true, true);
> +    virtio_set_queue_aio(vq, fn);
>      return 0;
>  }
>  
>  
> @@ -104,16 +136,19 @@ void virtio_scsi_dataplane_start(VirtIOSCSI *s)
>      }
>  
>      aio_context_acquire(s->ctx);
> -    rc = virtio_scsi_vring_init(s, vs->ctrl_vq, 0);
> +    rc = virtio_scsi_vring_init(s, vs->ctrl_vq, 0,
> +                             virtio_scsi_data_plane_handle_ctrl);

Tabs.

>      if (rc) {
>          goto fail_vrings;
>      }
> -    rc = virtio_scsi_vring_init(s, vs->event_vq, 1);
> +    rc = virtio_scsi_vring_init(s, vs->event_vq, 1,
> +                             virtio_scsi_data_plane_handle_event);

Tabs.

>      if (rc) {
>          goto fail_vrings;
>      }
>      for (i = 0; i < vs->conf.num_queues; i++) {
> -        rc = virtio_scsi_vring_init(s, vs->cmd_vqs[i], i + 2);
> +        rc = virtio_scsi_vring_init(s, vs->cmd_vqs[i], i + 2,
> +                                 virtio_scsi_data_plane_handle_cmd);

Tabs.

>          if (rc) {
>              goto fail_vrings;
>          }
> diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
> index 38f1e2c..30415c6 100644
> --- a/hw/scsi/virtio-scsi.c
> +++ b/hw/scsi/virtio-scsi.c
> @@ -374,7 +374,7 @@ fail:
>      return ret;
>  }
>  
> -void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
> +static void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
>  {
>      VirtIODevice *vdev = (VirtIODevice *)s;
>      uint32_t type;
> @@ -412,20 +412,28 @@ void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, 
> VirtIOSCSIReq *req)
>      }
>  }
>  
> -static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
> +void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq)
>  {
> -    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
>      VirtIOSCSIReq *req;
>  
> -    if (s->ctx && !s->dataplane_started) {
> -        virtio_scsi_dataplane_start(s);
> -        return;
> -    }
>      while ((req = virtio_scsi_pop_req(s, vq))) {
>          virtio_scsi_handle_ctrl_req(s, req);
>      }
>  }
>  
> +static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
> +{
> +    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
> +
> +    if (s->ctx) {
> +        virtio_scsi_dataplane_start(s);
> +        if (!s->dataplane_fenced) {
> +            return;
> +        }

Could be more readable if virtio_scsi_dataplane_start has a return value
indicating success.

> +    }
> +    virtio_scsi_handle_ctrl_vq(s, vq);
> +}
> +
>  static void virtio_scsi_complete_cmd_req(VirtIOSCSIReq *req)
>  {
>      /* Sense data is not in req->resp and is copied separately
> @@ -508,7 +516,7 @@ static void virtio_scsi_fail_cmd_req(VirtIOSCSIReq *req)
>      virtio_scsi_complete_cmd_req(req);
>  }
>  
> -bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
> +static bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq 
> *req)

Isn't this line too long now?

>  {
>      VirtIOSCSICommon *vs = &s->parent_obj;
>      SCSIDevice *d;
> @@ -550,7 +558,7 @@ bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, 
> VirtIOSCSIReq *req)
>      return true;
>  }
>  
> -void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
> +static void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq 
> *req)
>  {
>      SCSIRequest *sreq = req->sreq;
>      if (scsi_req_enqueue(sreq)) {
> @@ -560,17 +568,11 @@ void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, 
> VirtIOSCSIReq *req)
>      scsi_req_unref(sreq);
>  }
>  
> -static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
> +void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq)
>  {
> -    /* use non-QOM casts in the data path */
> -    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
>      VirtIOSCSIReq *req, *next;
>      QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
>  
> -    if (s->ctx && !s->dataplane_started) {
> -        virtio_scsi_dataplane_start(s);
> -        return;
> -    }
>      while ((req = virtio_scsi_pop_req(s, vq))) {
>          if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
>              QTAILQ_INSERT_TAIL(&reqs, req, next);
> @@ -582,6 +584,20 @@ static void virtio_scsi_handle_cmd(VirtIODevice *vdev, 
> VirtQueue *vq)
>      }
>  }
>  
> +static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
> +{
> +    /* use non-QOM casts in the data path */
> +    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
> +
> +    if (s->ctx) {
> +        virtio_scsi_dataplane_start(s);
> +        if (!s->dataplane_fenced) {
> +            return;
> +        }
> +    }
> +    virtio_scsi_handle_cmd_vq(s, vq);
> +}
> +
>  static void virtio_scsi_get_config(VirtIODevice *vdev,
>                                     uint8_t *config)
>  {
> @@ -725,17 +741,24 @@ out:
>      }
>  }
>  
> +void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq)
> +{
> +    if (s->events_dropped) {
> +        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
> +    }
> +}
> +
>  static void virtio_scsi_handle_event(VirtIODevice *vdev, VirtQueue *vq)
>  {
>      VirtIOSCSI *s = VIRTIO_SCSI(vdev);
>  
> -    if (s->ctx && !s->dataplane_started) {
> +    if (s->ctx) {
>          virtio_scsi_dataplane_start(s);
> -        return;
> -    }
> -    if (s->events_dropped) {
> -        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
> +        if (!s->dataplane_fenced) {
> +            return;
> +        }
>      }
> +    virtio_scsi_handle_event_vq(s, vq);
>  }
>  
>  static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense 
> sense)
> diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h
> index eef4e95..ba2f5ce 100644
> --- a/include/hw/virtio/virtio-scsi.h
> +++ b/include/hw/virtio/virtio-scsi.h
> @@ -139,9 +139,9 @@ void virtio_scsi_common_realize(DeviceState *dev, Error 
> **errp,
>                                  HandleOutput cmd);
>  
>  void virtio_scsi_common_unrealize(DeviceState *dev, Error **errp);
> -void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req);
> -bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req);
> -void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req);
> +void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq);
> +void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq);
> +void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq);
>  void virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq, VirtIOSCSIReq *req);
>  void virtio_scsi_free_req(VirtIOSCSIReq *req);
>  void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
> -- 
> 1.8.3.1
> 
>