Re: [Qemu-devel] [RFC PATCH v2 19/23] spapr: CPU hot unplug support

[David Gibson]

On Mon, Mar 23, 2015 at 07:06:00PM +0530, Bharata B Rao wrote:
> Support hot removal of CPU for sPAPR guests by sending the hot
> unplug notification to the guest via EPOW interrupt.
> 
> Signed-off-by: Bharata B Rao <address@hidden>
> ---
>  hw/ppc/spapr.c            | 78 
> ++++++++++++++++++++++++++++++++++++++++++++++-
>  linux-headers/linux/kvm.h |  1 +
>  target-ppc/kvm.c          |  7 +++++
>  target-ppc/kvm_ppc.h      |  6 ++++
>  4 files changed, 91 insertions(+), 1 deletion(-)
> 
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index b48994b..7b8784d 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu)
>      qemu_register_reset(spapr_cpu_reset, cpu);
>  }
>  
> +static void spapr_cpu_destroy(PowerPCCPU *cpu)
> +{
> +    xics_cpu_destroy(spapr->icp, cpu);
> +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> +}
> +
>  /* pSeries LPAR / sPAPR hardware init */
>  static void ppc_spapr_init(MachineState *machine)
>  {
> @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, 
> CPUState *cs, Error **errp)
>      }
>  }
>  
> +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs,
> +                                     Error **errp)
> +{
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> +
> +    drck->detach(drc, dev, NULL, NULL, errp);
> +}
> +
>  static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>                              Error **errp)
>  {
> @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler 
> *hotplug_dev, DeviceState *dev,
>      return;
>  }
>  
> +static int spapr_cpu_unplug(Object *obj, void *opaque)
> +{
> +    Error **errp = opaque;
> +    DeviceState *dev = DEVICE(obj);
> +    CPUState *cs = CPU(dev);
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    int smt = kvmppc_smt_threads();
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +
> +    spapr_cpu_destroy(cpu);

I may be misunderstanding something here, but don't you need to signal
the removal to the guest (and get its ack) before you "physically"
remove the cpu?

> +    /*
> +     * SMT threads return from here, only main thread (core) will
> +     * continue and signal hot unplug event to the guest.
> +     */
> +    if ((id % smt) != 0) {
> +        return 0;
> +    }
> +    g_assert(drc);
> +
> +    spapr_cpu_hotplug_remove(dev, cs, errp);
> +    if (*errp) {
> +        return -1;
> +    }
> +    spapr_hotplug_req_remove_event(drc);
> +
> +    return 0;
> +}
> +
> +static int spapr_cpu_core_unplug(Object *obj, void *opaque)
> +{
> +    Error **errp = opaque;
> +
> +    object_child_foreach(obj, spapr_cpu_unplug, errp);
> +    return 0;
> +}
> +
> +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev,
> +                            DeviceState *dev, Error **errp)
> +{
> +    object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp);
> +}
> +
>  static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>                                        DeviceState *dev, Error **errp)
>  {
> @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler 
> *hotplug_dev,
>      }
>  }
>  
> +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
> +                                      DeviceState *dev, Error **errp)
> +{
> +    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
> +        if (dev->hotplugged && spapr->dr_cpu_enabled) {
> +            spapr_cpu_socket_unplug(hotplug_dev, dev, errp);
> +        }
> +    }
> +}
> +
>  static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
>                                               DeviceState *dev)
>  {
> -    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
> +    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) ||
> +        object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
>          return HOTPLUG_HANDLER(machine);
>      }
>      return NULL;
> @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, 
> void *data)
>      mc->has_dynamic_sysbus = true;
>      mc->get_hotplug_handler = spapr_get_hotpug_handler;
>      hc->plug = spapr_machine_device_plug;
> +    hc->unplug = spapr_machine_device_unplug;
> +
>      smc->dr_phb_enabled = false;
>      smc->dr_cpu_enabled = false;
>      smc->dr_lmb_enabled = false;
> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> index 12045a1..0c1be07 100644
> --- a/linux-headers/linux/kvm.h
> +++ b/linux-headers/linux/kvm.h
> @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info {
>  #define KVM_CAP_PPC_FIXUP_HCALL 103
>  #define KVM_CAP_PPC_ENABLE_HCALL 104
>  #define KVM_CAP_CHECK_EXTENSION_VM 105
> +#define KVM_CAP_SPAPR_REUSE_VCPU 107

Updates to linux-headers/ are usually put into their own patch for
safety (along with an indication of what kernel commit the additions
came from).

>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
> index 1edf2b5..ee23bf6 100644
> --- a/target-ppc/kvm.c
> +++ b/target-ppc/kvm.c
> @@ -72,6 +72,7 @@ static int cap_ppc_watchdog;
>  static int cap_papr;
>  static int cap_htab_fd;
>  static int cap_fixup_hcalls;
> +static int cap_spapr_reuse_vcpu;

AFAICT nothing in this patch checks this variable.  Does it belong in
this patch?

>  static uint32_t debug_inst_opcode;
>  
> @@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s)
>       * only activated after this by kvmppc_set_papr() */
>      cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD);
>      cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL);
> +    cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU);
>  
>      if (!cap_interrupt_level) {
>          fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the 
> "
> @@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct 
> kvm_irq_routing_entry *route,
>  {
>      return 0;
>  }
> +
> +bool kvmppc_spapr_reuse_vcpu(void)
> +{
> +    return cap_spapr_reuse_vcpu;
> +}
> diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
> index 2e0224c..c831229 100644
> --- a/target-ppc/kvm_ppc.h
> +++ b/target-ppc/kvm_ppc.h
> @@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t 
> window_size, int *pfd,
>  int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
>  int kvmppc_reset_htab(int shift_hint);
>  uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
> +bool kvmppc_spapr_reuse_vcpu(void);
>  #endif /* !CONFIG_USER_ONLY */
>  bool kvmppc_has_cap_epr(void);
>  int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function);
> @@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env)
>      return 0;
>  }
>  
> +static inline bool kvmppc_spapr_reuse_vcpu(void)
> +{
> +    return false;
> +}
> +
>  #endif /* !CONFIG_USER_ONLY */
>  
>  static inline bool kvmppc_has_cap_epr(void)

-- 
David Gibson                    | I'll have my music baroque, and my code
david AT gibson.dropbear.id.au  | minimalist, thank you.  NOT _the_ _other_
                                | _way_ _around_!
http://www.ozlabs.org/~dgibson

pgpjkQIENJN5j.pgp
Description: PGP signature