Re: [Qemu-ppc] [PATCHv3 for-2.9 6/6] pseries: Allow HPT resizing with KVM

[David Gibson]

On Mon, Dec 19, 2016 at 03:07:43PM +1100, Suraj Jitindar Singh wrote:
> On Thu, 2016-12-15 at 17:11 +1100, David Gibson wrote:
> > So far, qemu implements the PAPR Hash Page Table (HPT) resizing
> > extension
> > with TCG.  The same implementation will work with KVM PR, but we
> > don't
> > currently allow that.  For KVM HV we can only implement resizing with
> > the
> > assistance of the host kernel, which needs a new capability and
> > ioctl()s.
> > 
> > This patch adds support for testing the new KVM capability and
> > implementing
> > the resize in terms of KVM facilities when necessary.  If we're
> > running on
> > a kernel which doesn't have the new capability flag at all, we fall
> > back to
> > testing for PR vs. HV KVM using the same hack that we already use in
> > a
> > number of places for older kernels.
> > 
> > NOTE: This patch updates the linux-headers tree with the define for
> > the
> > new capability and ioctl()s.  Since the corresponding kernel changes
> > aren't
> > yet upstream this is a temporary hack to be replaced by a proper
> > headers
> > update before merge.
> > 
> > Signed-off-by: David Gibson <address@hidden>
> > 
> > # Conflicts:
> > #   hw/ppc/spapr_hcall.c
> > #   target-ppc/kvm.c
> > ---
> >  hw/ppc/spapr_hcall.c      | 49 ++++++++++++++++++++++++++++++++
> >  linux-headers/linux/kvm.h | 10 +++++++
> >  target-ppc/kvm.c          | 71
> > +++++++++++++++++++++++++++++++++++++++++++++--
> >  target-ppc/kvm_ppc.h      | 14 ++++++++++
> >  4 files changed, 142 insertions(+), 2 deletions(-)
> > 
> > diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
> > index 3991c15..a42d0cb 100644
> > --- a/hw/ppc/spapr_hcall.c
> > +++ b/hw/ppc/spapr_hcall.c
> > @@ -460,6 +460,44 @@ static ram_addr_t get_current_ram_size(void)
> >      return size;
> >  }
> >  
> > +/* Convert a return code from the KVM ioctl()s implementing resize
> > HPT
> > + * into a PAPR hypercall return code */
> > +static target_ulong resize_hpt_convert_rc(int ret)
> > +{
> > +    if (ret >= 100000) {
> > +        return H_LONG_BUSY_ORDER_100_SEC;
> > +    } else if (ret >= 10000) {
> > +        return H_LONG_BUSY_ORDER_10_SEC;
> > +    } else if (ret >= 1000) {
> > +        return H_LONG_BUSY_ORDER_1_SEC;
> > +    } else if (ret >= 100) {
> > +        return H_LONG_BUSY_ORDER_100_MSEC;
> > +    } else if (ret >= 10) {
> > +        return H_LONG_BUSY_ORDER_10_MSEC;
> > +    } else if (ret > 0) {
> > +        return H_LONG_BUSY_ORDER_1_MSEC;
> > +    }
> > +
> > +    switch (ret) {
> > +    case 0:
> > +        return H_SUCCESS;
> > +    case -EPERM:
> > +        return H_AUTHORITY;
> > +    case -EINVAL:
> > +        return H_PARAMETER;
> > +    case -ENXIO:
> > +        return H_CLOSED;
> > +    case -ENOSPC:
> > +        return H_PTEG_FULL;
> > +    case -EBUSY:
> > +        return H_BUSY;
> > +    case -ENOMEM:
> > +        return H_NO_MEM;
> > +    default:
> > +        return H_HARDWARE;
> > +    }
> > +}
> > +
> >  static target_ulong h_resize_hpt_prepare(PowerPCCPU *cpu,
> >                                           sPAPRMachineState *spapr,
> >                                           target_ulong opcode,
> > @@ -468,6 +506,7 @@ static target_ulong
> > h_resize_hpt_prepare(PowerPCCPU *cpu,
> >      target_ulong flags = args[0];
> >      int shift = args[1];
> >      sPAPRPendingHPT *pending = spapr->pending_hpt;
> > +    int rc;
> >  
> >      if (spapr->resize_hpt == SPAPR_RESIZE_HPT_DISABLED) {
> >          return H_AUTHORITY;
> > @@ -491,6 +530,11 @@ static target_ulong
> > h_resize_hpt_prepare(PowerPCCPU *cpu,
> >          return H_RESOURCE;
> >      }
> >  
> > +    rc = kvmppc_resize_hpt_prepare(cpu, flags, shift);
> > +    if (rc != -ENOSYS) {
> > +        return resize_hpt_convert_rc(rc);
> > +    }
> > +
> >      if (pending) {
> >          /* something already in progress */
> >          if (pending->shift == shift) {
> > @@ -696,6 +740,11 @@ static target_ulong
> > h_resize_hpt_commit(PowerPCCPU *cpu,
> >  
> >      trace_spapr_h_resize_hpt_commit(flags, shift);
> >  
> > +    rc = kvmppc_resize_hpt_commit(cpu, flags, shift);
> > +    if (rc != -ENOSYS) {
> > +        return resize_hpt_convert_rc(rc);
> > +    }
> > +
> >      if (flags != 0) {
> >          return H_PARAMETER;
> >      }
> > diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> > index 4806e06..f11b6ac 100644
> > --- a/linux-headers/linux/kvm.h
> > +++ b/linux-headers/linux/kvm.h
> > @@ -684,6 +684,12 @@ struct kvm_ppc_smmu_info {
> >  
> >  #define KVM_PPC_PVINFO_FLAGS_EV_IDLE   (1<<0)
> >  
> > +/* for KVM_PPC_RESIZE_HPT_{PREPARE,COMMIT} */
> > +struct kvm_ppc_resize_hpt {
> > +        __u64 flags;
> > +        __u32 shift;
> > +};
> > +
> >  #define KVMIO 0xAE
> >  
> >  /* machine type bits, to be used as argument to KVM_CREATE_VM */
> > @@ -870,6 +876,7 @@ struct kvm_ppc_smmu_info {
> >  #define KVM_CAP_S390_USER_INSTR0 130
> >  #define KVM_CAP_MSI_DEVID 131
> >  #define KVM_CAP_PPC_HTM 132
> > +#define KVM_CAP_SPAPR_RESIZE_HPT 133
> >  
> >  #ifdef KVM_CAP_IRQ_ROUTING
> >  
> > @@ -1179,6 +1186,9 @@ struct kvm_s390_ucas_mapping {
> >  #define KVM_ARM_SET_DEVICE_ADDR      _IOW(KVMIO,  0xab, struct
> > kvm_arm_device_addr)
> >  /* Available with KVM_CAP_PPC_RTAS */
> >  #define KVM_PPC_RTAS_DEFINE_TOKEN _IOW(KVMIO,  0xac, struct
> > kvm_rtas_token_args)
> > +/* Available with KVM_CAP_SPAPR_RESIZE_HPT */
> > +#define KVM_PPC_RESIZE_HPT_PREPARE _IOR(KVMIO, 0xad, struct
> > kvm_ppc_resize_hpt)
> > +#define KVM_PPC_RESIZE_HPT_COMMIT _IOR(KVMIO, 0xae, struct
> > kvm_ppc_resize_hpt)
> >  
> >  /* ioctl for vm fd */
> >  #define KVM_CREATE_DEVICE    _IOWR(KVMIO,  0xe0, struct
> > kvm_create_device)
> > diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
> > index f015d56..53e143c 100644
> > --- a/target-ppc/kvm.c
> > +++ b/target-ppc/kvm.c
> > @@ -82,6 +82,7 @@ static int cap_papr;
> >  static int cap_htab_fd;
> >  static int cap_fixup_hcalls;
> >  static int cap_htm;             /* Hardware transactional memory
> > support */
> > +static int cap_resize_hpt;
> >  
> >  static uint32_t debug_inst_opcode;
> >  
> > @@ -135,6 +136,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
> >      cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD);
> >      cap_fixup_hcalls = kvm_check_extension(s,
> > KVM_CAP_PPC_FIXUP_HCALL);
> >      cap_htm = kvm_vm_check_extension(s, KVM_CAP_PPC_HTM);
> > +    cap_resize_hpt = kvm_vm_check_extension(s,
> > KVM_CAP_SPAPR_RESIZE_HPT);
> >  
> >      if (!cap_interrupt_level) {
> >          fprintf(stderr, "KVM: Couldn't find level irq capability.
> > Expect the "
> > @@ -2677,10 +2679,75 @@ int kvmppc_enable_hwrng(void)
> >  void kvmppc_check_papr_resize_hpt(Error **errp)
> >  {
> >      if (!kvm_enabled()) {
> > +        return; /* No KVM, we're good */
> > +    }
> > +
> > +    switch (cap_resize_hpt) {
> Is it worth #defining these values somewhere for clarity?

Hm, maybe.  It would need to go in the actual kernel headers.

I'm trying to find a less weird encoding of the information we need
here.  We'll see how we go.

> > +    case 0:
> > +        /*
> > +         * Old kernel.  We can only do resizing if the HPT is under
> > +         * qemu control.  However, to size the HPT correctly we need
> > +         * to call this function before we naturally know whether or
> > +         * not we'll have an in-kernel HPT.  So we have to fall back
> > +         * on checking PR vs HV KVM - resizing is possible with PR
> > +         * only.
> > +         */
> > +        if (!kvmppc_is_pr(kvm_state)) {
> > +            error_setg(errp, "Hash page table resizing not available
> > with this KVM version");
> > +        }
> > +        return;
> > +
> > +    case 1:
> > +        /*
> > +         * If we have a kernel new enough to advertise this
> > +         * capability, then it also correctly advertises
> > +         * KVM_CAP_PPC_ALLOC_HTAB according to whether it will be
> > +         * using an in-kernel HPT or not (older kernels didn't).
> > +         */
> > +        if (kvm_vm_check_extension(kvm_state,
> > KVM_CAP_PPC_ALLOC_HTAB)) {
> > +            error_setg(errp,
> > +                       "Hash page table resizing not available with
> > kernel managed HPT");
> > +        }
> > +        /* qemu managed HPT, we're good */
> > +        return;
> > +
> > +    case 2:
> > +        /* HPT resize allowed via explicit ioctl()s */
> > +        return;
> > +
> > +    default:
> > +        error_setg(errp, "Unknown KVM_CAP_SPAPR_RESIZE_HPT value
> > %d",
> > +                   cap_resize_hpt);
> >          return;
> >      }
> > +}
> >  
> > -    /* TODO: Check for resize-capable KVM implementations */
> > +int kvmppc_resize_hpt_prepare(PowerPCCPU *cpu, target_ulong flags,
> > int shift)
> > +{
> > +    CPUState *cs = CPU(cpu);
> > +    struct kvm_ppc_resize_hpt rhpt = {
> > +        .flags = flags,
> > +        .shift = shift,
> > +    };
> > +
> > +    if (cap_resize_hpt != 2) {
> > +        return -ENOSYS;
> > +    }
> > +
> > +    return kvm_vm_ioctl(cs->kvm_state, KVM_PPC_RESIZE_HPT_PREPARE,
> > &rhpt);
> > +}
> > +
> > +int kvmppc_resize_hpt_commit(PowerPCCPU *cpu, target_ulong flags,
> > int shift)
> > +{
> > +    CPUState *cs = CPU(cpu);
> > +    struct kvm_ppc_resize_hpt rhpt = {
> > +        .flags = flags,
> > +        .shift = shift,
> > +    };
> > +
> > +    if (cap_resize_hpt != 2) {
> > +        return -ENOSYS;
> > +    }
> >  
> > -    error_setg(errp, "Hash page table resizing not available with
> > this KVM version");
> > +    return kvm_vm_ioctl(cs->kvm_state, KVM_PPC_RESIZE_HPT_COMMIT,
> > &rhpt);
> >  }
> > diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
> > index 3e852ba..a3e3b1e 100644
> > --- a/target-ppc/kvm_ppc.h
> > +++ b/target-ppc/kvm_ppc.h
> > @@ -60,6 +60,8 @@ int kvmppc_enable_hwrng(void);
> >  int kvmppc_put_books_sregs(PowerPCCPU *cpu);
> >  PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void);
> >  void kvmppc_check_papr_resize_hpt(Error **errp);
> > +int kvmppc_resize_hpt_prepare(PowerPCCPU *cpu, target_ulong flags,
> > int shift);
> > +int kvmppc_resize_hpt_commit(PowerPCCPU *cpu, target_ulong flags,
> > int shift);
> >  
> >  #else
> >  
> > @@ -275,6 +277,18 @@ static inline void
> > kvmppc_check_papr_resize_hpt(Error **errp)
> >  {
> >      return;
> >  }
> > +
> > +static inline int kvmppc_resize_hpt_prepare(PowerPCCPU *cpu,
> > +                                            target_ulong flags, int
> > shift)
> > +{
> > +    return -ENOSYS;
> > +}
> > +
> > +static inline int kvmppc_resize_hpt_commit(PowerPCCPU *cpu,
> > +                                           target_ulong flags, int
> > shift)
> > +{
> > +    return -ENOSYS;
> > +}
> >  #endif
> >  
> >  #ifndef CONFIG_KVM
> 

-- 
David Gibson                    | I'll have my music baroque, and my code
david AT gibson.dropbear.id.au  | minimalist, thank you.  NOT _the_ _other_
                                | _way_ _around_!
http://www.ozlabs.org/~dgibson

signature.asc
Description: PGP signature