Re: [PATCH v5 3/6] net/vmnet: implement shared mode (vmnet-shared)

[Jason Wang]

On Fri, Nov 19, 2021 at 1:12 AM Vladislav Yaroshchuk
<yaroshchuk2000@gmail.com> wrote:
>
>
>
> пн, 15 нояб. 2021 г. в 07:47, Jason Wang <jasowang@redhat.com>:
>>
>> On Fri, Nov 12, 2021 at 5:14 PM Vladislav Yaroshchuk
>> <yaroshchuk2000@gmail.com> wrote:
>> >
>> > Signed-off-by: Phillip Tennen <phillip@axleos.com>
>> > Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
>> > ---
>>
>> Commit log please.
>>
>>
>
> Sorry, I don't understand what you mean here.
> What is the 'commit log'?

I meant the change log to describe the changes.

>
>>
>> >  net/vmnet-common.m | 305 +++++++++++++++++++++++++++++++++++++++++++++
>> >  net/vmnet-shared.c |  75 ++++++++++-
>> >  net/vmnet_int.h    |  23 ++++
>> >  3 files changed, 399 insertions(+), 4 deletions(-)
>> >
>> > diff --git a/net/vmnet-common.m b/net/vmnet-common.m
>> > index 532d152840..b058e1b846 100644
>> > --- a/net/vmnet-common.m
>> > +++ b/net/vmnet-common.m
>> > @@ -10,6 +10,8 @@
>> >   */
>> >
>> >  #include "qemu/osdep.h"
>> > +#include "qemu/main-loop.h"
>> > +#include "qemu/log.h"
>> >  #include "qapi/qapi-types-net.h"
>> >  #include "vmnet_int.h"
>> >  #include "clients.h"
>> > @@ -17,4 +19,307 @@
>> >  #include "qapi/error.h"
>> >
>> >  #include <vmnet/vmnet.h>
>> > +#include <dispatch/dispatch.h>
>> >
>> > +#ifdef DEBUG
>> > +#define D(x) x
>> > +#define D_LOG(...) qemu_log(__VA_ARGS__)
>> > +#else
>> > +#define D(x) do { } while (0)
>> > +#define D_LOG(...) do { } while (0)
>> > +#endif
>> > +
>> > +typedef struct vmpktdesc vmpktdesc_t;
>> > +typedef struct iovec iovec_t;
>> > +
>> > +static void vmnet_set_send_enabled(VmnetCommonState *s, bool enable)
>> > +{
>> > +    s->send_enabled = enable;
>>
>> Is there a way to disable the event when enable is false?
>>
>
> It seems there's no way except setting/unsetting
> the callback via `vmnet_interface_set_event_callback`.
> I decided to drop packages using `s->send_enabled`
> without dealing with the callback.

ok.

>
>> > +}
>> > +
>> > +
>> > +static void vmnet_send_completed(NetClientState *nc, ssize_t len)
>> > +{
>> > +    VmnetCommonState *s = DO_UPCAST(VmnetCommonState, nc, nc);
>> > +    vmnet_set_send_enabled(s, true);
>> > +}
>> > +
>> > +
>> > +static void vmnet_send(NetClientState *nc,
>> > +                       interface_event_t event_id,
>> > +                       xpc_object_t event)
>> > +{
>> > +    assert(event_id == VMNET_INTERFACE_PACKETS_AVAILABLE);
>> > +
>> > +    VmnetCommonState *s;
>> > +    uint64_t packets_available;
>> > +
>> > +    struct iovec *iov;
>> > +    struct vmpktdesc *packets;
>> > +    int pkt_cnt;
>> > +    int i;
>> > +
>> > +    vmnet_return_t if_status;
>> > +    ssize_t size;
>> > +
>> > +    s = DO_UPCAST(VmnetCommonState, nc, nc);
>> > +
>> > +    packets_available = xpc_dictionary_get_uint64(
>> > +        event,
>> > +        vmnet_estimated_packets_available_key
>> > +    );
>> > +
>> > +    pkt_cnt = (packets_available < VMNET_PACKETS_LIMIT) ?
>> > +              packets_available :
>> > +              VMNET_PACKETS_LIMIT;
>> > +
>> > +
>> > +    iov = s->iov_buf;
>> > +    packets = s->packets_buf;
>> > +
>> > +    for (i = 0; i < pkt_cnt; ++i) {
>> > +        packets[i].vm_pkt_size = s->max_packet_size;
>> > +        packets[i].vm_pkt_iovcnt = 1;
>> > +        packets[i].vm_flags = 0;
>> > +    }
>> > +
>> > +    if_status = vmnet_read(s->vmnet_if, packets, &pkt_cnt);
>> > +    if (if_status != VMNET_SUCCESS) {
>> > +        error_printf("vmnet: read failed: %s\n",
>> > +                     vmnet_status_map_str(if_status));
>> > +    }
>> > +    qemu_mutex_lock_iothread();
>> > +    for (i = 0; i < pkt_cnt; ++i) {
>> > +        size = qemu_send_packet_async(nc,
>> > +                                      iov[i].iov_base,
>> > +                                      packets[i].vm_pkt_size,
>> > +                                      vmnet_send_completed);
>> > +        if (size == 0) {
>> > +            vmnet_set_send_enabled(s, false);
>> > +        } else if (size < 0) {
>> > +            break;
>> > +        }
>> > +    }
>> > +    qemu_mutex_unlock_iothread();
>> > +
>> > +}
>> > +
>> > +
>> > +static void vmnet_register_event_callback(VmnetCommonState *s)
>> > +{
>> > +    dispatch_queue_t avail_pkt_q = dispatch_queue_create(
>> > +        "org.qemu.vmnet.if_queue",
>> > +        DISPATCH_QUEUE_SERIAL
>> > +    );
>> > +
>> > +    vmnet_interface_set_event_callback(
>> > +        s->vmnet_if,
>> > +        VMNET_INTERFACE_PACKETS_AVAILABLE,
>> > +        avail_pkt_q,
>> > +        ^(interface_event_t event_id, xpc_object_t event) {
>> > +          if (s->send_enabled) {
>> > +              vmnet_send(&s->nc, event_id, event);
>> > +          }
>> > +        });
>> > +}
>> > +
>> > +
>> > +static void vmnet_bufs_init(VmnetCommonState *s)
>> > +{
>> > +    int i;
>> > +    struct vmpktdesc *packets;
>> > +    struct iovec *iov;
>> > +
>> > +    packets = s->packets_buf;
>> > +    iov = s->iov_buf;
>> > +
>> > +    for (i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
>> > +        iov[i].iov_len = s->max_packet_size;
>> > +        iov[i].iov_base = g_malloc0(iov[i].iov_len);
>> > +        packets[i].vm_pkt_iov = iov + i;
>> > +    }
>> > +}
>> > +
>> > +
>> > +const char *vmnet_status_map_str(vmnet_return_t status)
>> > +{
>> > +    switch (status) {
>> > +    case VMNET_SUCCESS:
>> > +        return "success";
>> > +    case VMNET_FAILURE:
>> > +        return "general failure";
>> > +    case VMNET_MEM_FAILURE:
>> > +        return "memory allocation failure";
>> > +    case VMNET_INVALID_ARGUMENT:
>> > +        return "invalid argument specified";
>> > +    case VMNET_SETUP_INCOMPLETE:
>> > +        return "interface setup is not complete";
>> > +    case VMNET_INVALID_ACCESS:
>> > +        return "invalid access, permission denied";
>> > +    case VMNET_PACKET_TOO_BIG:
>> > +        return "packet size is larger than MTU";
>> > +    case VMNET_BUFFER_EXHAUSTED:
>> > +        return "buffers exhausted in kernel";
>> > +    case VMNET_TOO_MANY_PACKETS:
>> > +        return "packet count exceeds limit";
>> > +    case VMNET_SHARING_SERVICE_BUSY:
>> > +        return "conflict, sharing service is in use";
>> > +    default:
>> > +        return "unknown vmnet error";
>> > +    }
>> > +}
>> > +
>> > +
>> > +int vmnet_if_create(NetClientState *nc,
>> > +                    xpc_object_t if_desc,
>> > +                    Error **errp,
>> > +                    void (*completion_callback)(xpc_object_t 
>> > interface_param))
>> > +{
>> > +    VmnetCommonState *s;
>> > +
>> > +    dispatch_queue_t if_create_q;
>> > +    dispatch_semaphore_t if_created_sem;
>> > +
>> > +    __block vmnet_return_t if_status;
>> > +
>> > +    if_create_q = dispatch_queue_create("org.qemu.vmnet.create",
>> > +                                        DISPATCH_QUEUE_SERIAL);
>> > +    if_created_sem = dispatch_semaphore_create(0);
>> > +
>> > +    xpc_dictionary_set_bool(
>> > +        if_desc,
>> > +        vmnet_allocate_mac_address_key,
>> > +        false
>> > +    );
>> > +
>> > +    D(D_LOG("vmnet.start.interface_desc:\n");
>> > +    xpc_dictionary_apply(if_desc,
>> > +                         ^bool(const char *k, xpc_object_t v) {
>> > +                           char *desc = xpc_copy_description(v);
>> > +                           D_LOG("  %s=%s\n", k, desc);
>> > +                           free(desc);
>> > +                           return true;
>> > +                         }));
>> > +
>> > +    s = DO_UPCAST(VmnetCommonState, nc, nc);
>> > +    s->vmnet_if = vmnet_start_interface(
>> > +        if_desc,
>> > +        if_create_q,
>> > +        ^(vmnet_return_t status, xpc_object_t interface_param) {
>> > +          if_status = status;
>> > +          if (status != VMNET_SUCCESS || !interface_param) {
>> > +              dispatch_semaphore_signal(if_created_sem);
>> > +              return;
>> > +          }
>> > +
>> > +          s->mtu = xpc_dictionary_get_uint64(
>> > +              interface_param,
>> > +              vmnet_mtu_key);
>> > +          s->max_packet_size = xpc_dictionary_get_uint64(
>> > +              interface_param,
>> > +              vmnet_max_packet_size_key);
>> > +          D(D_LOG("vmnet.start.interface_param:\n");
>> > +          xpc_dictionary_apply(interface_param,
>> > +                               ^bool(const char *k, xpc_object_t v) {
>> > +                                 char *desc = xpc_copy_description(v);
>> > +                                 D_LOG("  %s=%s\n", k, desc);
>> > +                                 free(desc);
>> > +                                 return true;
>> > +                               }));
>> > +          dispatch_semaphore_signal(if_created_sem);
>> > +        });
>> > +
>> > +    if (s->vmnet_if == NULL) {
>> > +        error_setg(errp, "unable to create interface with requested 
>> > params");
>> > +        return -1;
>> > +    }
>> > +
>> > +    dispatch_semaphore_wait(if_created_sem, DISPATCH_TIME_FOREVER);
>> > +    dispatch_release(if_create_q);
>> > +
>> > +    if (if_status != VMNET_SUCCESS) {
>> > +        error_setg(errp,
>> > +                   "cannot create vmnet interface: %s",
>> > +                   vmnet_status_map_str(if_status));
>> > +        return -1;
>> > +    }
>> > +
>> > +    vmnet_register_event_callback(s);
>> > +    vmnet_bufs_init(s);
>> > +    vmnet_set_send_enabled(s, true);
>> > +
>> > +    return 0;
>> > +}
>> > +
>> > +
>> > +ssize_t vmnet_receive_common(NetClientState *nc,
>> > +                             const uint8_t *buf,
>> > +                             size_t size)
>> > +{
>> > +    VmnetCommonState *s;
>> > +    vmpktdesc_t packet;
>> > +    iovec_t iov;
>> > +    int pkt_cnt;
>> > +    vmnet_return_t if_status;
>> > +
>> > +    s = DO_UPCAST(VmnetCommonState, nc, nc);
>> > +
>> > +    if (size > s->max_packet_size) {
>> > +        warn_report("vmnet: packet is too big, %zu > %llu\n",
>> > +                    packet.vm_pkt_size,
>> > +                    s->max_packet_size);
>> > +        return -1;
>> > +    }
>> > +
>> > +    iov.iov_base = (char *) buf;
>> > +    iov.iov_len = size;
>> > +
>> > +    packet.vm_pkt_iovcnt = 1;
>> > +    packet.vm_flags = 0;
>> > +    packet.vm_pkt_size = size;
>> > +    packet.vm_pkt_iov = &iov;
>>
>> It looks to me vmnet framework supports iov so I wonder if a
>> .receive_iov() is better because of its performance.
>>
>
> I've just tried to implement this call, and because of some
> reason `vmnet_write` fails with `VMNET_INVALID_ARGUMENT`
> when iovcnt > 1. Tested with `vmxnet3`. Collecting all the iovs
> into a big one and passing it to vmnet works fine (the default
> behaviour when only the .receive() but not the .receive_iov()
> is implemented).
>
> This should be investigated, but currently I don't understand
> what exactly causes this error. The fact that vmnet.framework
> is a 'blackbox' makes the situation much worse.
>
> Phillip's version is also broken:
> https://patchew.org/QEMU/20210218134947.1860-1-phillip.ennen@gmail.com/
> Looks like this wasn't noticed before.
>
> If it's applicable, we can use the .receive() only, and put
> .receive_iov() implementation to the backlog.

Ok, we can go with receive() first.

Thanks

>
>> > +
>> > +    pkt_cnt = 1;
>> > +    if_status = vmnet_write(s->vmnet_if, &packet, &pkt_cnt);
>> > +
>> > +    if (if_status != VMNET_SUCCESS) {
>> > +        error_report("vmnet: write error: %s\n",
>> > +                     vmnet_status_map_str(if_status));
>> > +    }
>> > +
>> > +    if (if_status == VMNET_SUCCESS && pkt_cnt) {
>> > +        return size;
>> > +    }
>> > +    return 0;
>> > +}
>> > +
>> > +
>> > +void vmnet_cleanup_common(NetClientState *nc)
>> > +{
>> > +    VmnetCommonState *s;
>> > +    dispatch_queue_t if_destroy_q;
>> > +
>> > +    s = DO_UPCAST(VmnetCommonState, nc, nc);
>> > +
>> > +    qemu_purge_queued_packets(nc);
>> > +    vmnet_set_send_enabled(s, false);
>> > +
>> > +    if (s->vmnet_if == NULL) {
>> > +        return;
>> > +    }
>> > +
>> > +    if_destroy_q = dispatch_queue_create(
>> > +        "org.qemu.vmnet.destroy",
>> > +        DISPATCH_QUEUE_SERIAL
>> > +    );
>> > +
>> > +    vmnet_stop_interface(
>> > +        s->vmnet_if,
>> > +        if_destroy_q,
>> > +        ^(vmnet_return_t status) {
>> > +        });
>> > +
>> > +    for (int i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
>> > +        g_free(s->iov_buf[i].iov_base);
>> > +    }
>> > +}
>> > diff --git a/net/vmnet-shared.c b/net/vmnet-shared.c
>> > index f8c4a4f3b8..b27ada3219 100644
>> > --- a/net/vmnet-shared.c
>> > +++ b/net/vmnet-shared.c
>> > @@ -10,16 +10,83 @@
>> >
>> >  #include "qemu/osdep.h"
>> >  #include "qapi/qapi-types-net.h"
>> > +#include "qapi/error.h"
>> >  #include "vmnet_int.h"
>> >  #include "clients.h"
>> > -#include "qemu/error-report.h"
>> > -#include "qapi/error.h"
>> >
>> >  #include <vmnet/vmnet.h>
>> >
>> > +typedef struct VmnetSharedState {
>> > +  VmnetCommonState cs;
>> > +} VmnetSharedState;
>> > +
>> > +
>> > +static xpc_object_t create_if_desc(const Netdev *netdev, Error **errp)
>> > +{
>> > +    const NetdevVmnetSharedOptions *options = &(netdev->u.vmnet_shared);
>> > +    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
>> > +
>> > +    xpc_dictionary_set_uint64(
>> > +        if_desc,
>> > +        vmnet_operation_mode_key,
>> > +        VMNET_SHARED_MODE
>> > +    );
>> > +
>> > +    xpc_dictionary_set_bool(
>> > +        if_desc,
>> > +        vmnet_enable_isolation_key,
>> > +        options->isolated
>> > +    );
>> > +
>> > +    if (options->has_nat66_prefix) {
>> > +        xpc_dictionary_set_string(if_desc,
>> > +                                  vmnet_nat66_prefix_key,
>> > +                                  options->nat66_prefix);
>> > +    }
>> > +
>> > +    if (options->has_dhcpstart ||
>> > +        options->has_dhcpend ||
>> > +        options->has_subnet_mask) {
>> > +
>> > +        if (options->has_dhcpstart &&
>> > +            options->has_dhcpend &&
>> > +            options->has_subnet_mask) {
>> > +
>> > +            xpc_dictionary_set_string(if_desc,
>> > +                                      vmnet_start_address_key,
>> > +                                      options->dhcpstart);
>> > +            xpc_dictionary_set_string(if_desc,
>> > +                                      vmnet_end_address_key,
>> > +                                      options->dhcpend);
>> > +            xpc_dictionary_set_string(if_desc,
>> > +                                      vmnet_subnet_mask_key,
>> > +                                      options->subnet_mask);
>> > +        } else {
>> > +            error_setg(
>> > +                errp,
>> > +                "'dhcpstart', 'dhcpend', 'subnet_mask' "
>> > +                "should be provided together"
>> > +            );
>> > +        }
>> > +    }
>> > +
>> > +    return if_desc;
>> > +}
>> > +
>> > +static NetClientInfo net_vmnet_shared_info = {
>> > +    .type = NET_CLIENT_DRIVER_VMNET_SHARED,
>> > +    .size = sizeof(VmnetSharedState),
>> > +    .receive = vmnet_receive_common,
>> > +    .cleanup = vmnet_cleanup_common,
>> > +};
>> > +
>> >  int net_init_vmnet_shared(const Netdev *netdev, const char *name,
>> >                            NetClientState *peer, Error **errp)
>> >  {
>> > -  error_setg(errp, "vmnet-shared is not implemented yet");
>> > -  return -1;
>> > +    NetClientState *nc = qemu_new_net_client(&net_vmnet_shared_info,
>> > +                                             peer, "vmnet-shared", name);
>> > +    xpc_object_t if_desc = create_if_desc(netdev, errp);
>> > +
>> > +    return vmnet_if_create(nc, if_desc, errp, NULL);
>> >  }
>> > +
>> > diff --git a/net/vmnet_int.h b/net/vmnet_int.h
>> > index c5982259a4..3979fe4678 100644
>> > --- a/net/vmnet_int.h
>> > +++ b/net/vmnet_int.h
>> > @@ -16,10 +16,33 @@
>> >
>> >  #include <vmnet/vmnet.h>
>> >
>> > +#define VMNET_PACKETS_LIMIT 50
>> > +
>> >  typedef struct VmnetCommonState {
>> >    NetClientState nc;
>> > +  interface_ref vmnet_if;
>> > +
>> > +  bool send_enabled;
>> > +
>> > +  uint64_t mtu;
>> > +  uint64_t max_packet_size;
>> > +
>> > +  struct vmpktdesc packets_buf[VMNET_PACKETS_LIMIT];
>> > +  struct iovec iov_buf[VMNET_PACKETS_LIMIT];
>> >
>> >  } VmnetCommonState;
>> >
>> > +const char *vmnet_status_map_str(vmnet_return_t status);
>> > +
>> > +int vmnet_if_create(NetClientState *nc,
>> > +                    xpc_object_t if_desc,
>> > +                    Error **errp,
>> > +                    void (*completion_callback)(xpc_object_t 
>> > interface_param));
>> > +
>> > +ssize_t vmnet_receive_common(NetClientState *nc,
>> > +                             const uint8_t *buf,
>> > +                             size_t size);
>> > +
>> > +void vmnet_cleanup_common(NetClientState *nc);
>> >
>> >  #endif /* VMNET_INT_H */
>> > --
>> > 2.23.0
>> >
>
>
> --
> Best Regards,
>
> Vladislav Yaroshchuk