Re: [PATCH v6 10/13] i386: Check for apic id encoding

[Babu Moger]

On 3/11/20 10:48 AM, Igor Mammedov wrote:
> On Wed, 11 Mar 2020 09:50:04 -0500
> Babu Moger <address@hidden> wrote:
> 
>> On 3/11/20 7:17 AM, Igor Mammedov wrote:
>>> On Tue, 10 Mar 2020 21:34:25 -0500
>>> Babu Moger <address@hidden> wrote:
>>>   
>>>> Check X86CPUDefinition if use_epyc_apic_id_encoding is enabled. If enabled
>>>> update X86MachineState with EPYC mode apic_id encoding handlers.
>>>>
>>>> Also update the calling convention to use apic_id handlers from 
>>>> X86MachineState.
>>>>
>>>> Signed-off-by: Babu Moger <address@hidden>
>>>> Acked-by: Michael S. Tsirkin <address@hidden>  
>>> usually tags are not retained in case patch is changed significantly
>>>   
>> Ok.
>>
>>> see more below
>>>   
>>>> ---
>>>>  hw/i386/pc.c      |    6 +++---
>>>>  hw/i386/x86.c     |   32 +++++++++++++++++++++++++++-----
>>>>  target/i386/cpu.c |   11 +++++++++++
>>>>  target/i386/cpu.h |    1 +
>>>>  4 files changed, 42 insertions(+), 8 deletions(-)
>>>>
>>>> diff --git a/hw/i386/pc.c b/hw/i386/pc.c
>>>> index 98ee763f68..2d7d611184 100644
>>>> --- a/hw/i386/pc.c
>>>> +++ b/hw/i386/pc.c
>>>> @@ -1580,14 +1580,14 @@ static void pc_cpu_pre_plug(HotplugHandler 
>>>> *hotplug_dev,
>>>>          topo_ids.die_id = cpu->die_id;
>>>>          topo_ids.core_id = cpu->core_id;
>>>>          topo_ids.smt_id = cpu->thread_id;
>>>> -        cpu->apic_id = x86_apicid_from_topo_ids(&topo_info, &topo_ids);
>>>> +        cpu->apic_id = x86ms->apicid_from_topo_ids(&topo_info, &topo_ids);
>>>>      }
>>>>  
>>>>      cpu_slot = pc_find_cpu_slot(MACHINE(pcms), cpu->apic_id, &idx);
>>>>      if (!cpu_slot) {
>>>>          MachineState *ms = MACHINE(pcms);
>>>>  
>>>> -        x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids);
>>>> +        x86ms->topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids);
>>>>          error_setg(errp,
>>>>              "Invalid CPU [socket: %u, die: %u, core: %u, thread: %u] with"
>>>>              " APIC ID %" PRIu32 ", valid index range 0:%d",
>>>> @@ -1608,7 +1608,7 @@ static void pc_cpu_pre_plug(HotplugHandler 
>>>> *hotplug_dev,
>>>>      /* TODO: move socket_id/core_id/thread_id checks into 
>>>> x86_cpu_realizefn()
>>>>       * once -smp refactoring is complete and there will be CPU private
>>>>       * CPUState::nr_cores and CPUState::nr_threads fields instead of 
>>>> globals */
>>>> -    x86_topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids);
>>>> +    x86ms->topo_ids_from_apicid(cpu->apic_id, &topo_info, &topo_ids);
>>>>      if (cpu->socket_id != -1 && cpu->socket_id != topo_ids.pkg_id) {
>>>>          error_setg(errp, "property socket-id: %u doesn't match set 
>>>> apic-id:"
>>>>              " 0x%x (socket-id: %u)", cpu->socket_id, cpu->apic_id,
>>>> diff --git a/hw/i386/x86.c b/hw/i386/x86.c
>>>> index 7dc237c014..ad85347142 100644
>>>> --- a/hw/i386/x86.c
>>>> +++ b/hw/i386/x86.c
>>>> @@ -68,6 +68,25 @@ inline void init_topo_info(X86CPUTopoInfo *topo_info,
>>>>      topo_info->threads_per_core = ms->smp.threads;
>>>>  }
>>>>  
>>>> +/*
>>>> + * Check for APIC ID encoding
>>>> + *
>>>> + * AMD uses different apic id encoding for their EPYC based cpus.
>>>> + * Check if we need to use different handlers than the default.  
>>> comment is not valid, cpu_x86_use_epyc_apic_id_encoding() does the checking
>>> but this function actually overrides it.
>>>
>>> it would be better to name it properly and amend comment, something like
>>>
>>> if (cpu_x86_use_epyc_apic_id_encoding(machine->cpu_type)) {
>>>    set_epyc_topo_handlers()
>>> }  
>>
>> Ok. Will fix it. I will move the assignment to new function
>> set_epyc_topo_handlers() and fix the comment.
>>
>>>   
>>>> + */
>>>> +static void x86_check_apic_id_encoding(MachineState *machine)
>>>> +{
>>>> +    X86MachineState *x86ms = X86_MACHINE(machine);
>>>> +
>>>> +    if (cpu_x86_use_epyc_apic_id_encoding(machine->cpu_type)) {
>>>> +        x86ms->apicid_from_cpu_idx = x86_apicid_from_cpu_idx_epyc;
>>>> +        x86ms->topo_ids_from_apicid = x86_topo_ids_from_apicid_epyc;
>>>> +        x86ms->topo_ids_from_idx = x86_topo_ids_from_idx_epyc;
>>>> +        x86ms->apicid_from_topo_ids = x86_apicid_from_topo_ids_epyc;
>>>> +        x86ms->apicid_pkg_offset = apicid_pkg_offset_epyc;
>>>> +    }
>>>> +}
>>>> +
>>>>  /*
>>>>   * Calculates initial APIC ID for a specific CPU index
>>>>   *
>>>> @@ -86,7 +105,7 @@ uint32_t x86_cpu_apic_id_from_index(X86MachineState 
>>>> *x86ms,
>>>>  
>>>>      init_topo_info(&topo_info, x86ms);
>>>>  
>>>> -    correct_id = x86_apicid_from_cpu_idx(&topo_info, cpu_index);
>>>> +    correct_id = x86ms->apicid_from_cpu_idx(&topo_info, cpu_index);
>>>>      if (x86mc->compat_apic_id_mode) {
>>>>          if (cpu_index != correct_id && !warned && !qtest_enabled()) {
>>>>              error_report("APIC IDs set in compatibility mode, "
>>>> @@ -158,8 +177,8 @@ int64_t x86_get_default_cpu_node_id(const MachineState 
>>>> *ms, int idx)
>>>>     init_topo_info(&topo_info, x86ms);
>>>>  
>>>>     assert(idx < ms->possible_cpus->len);
>>>> -   x86_topo_ids_from_apicid(ms->possible_cpus->cpus[idx].arch_id,
>>>> -                            &topo_info, &topo_ids);
>>>> +   x86ms->topo_ids_from_apicid(ms->possible_cpus->cpus[idx].arch_id,
>>>> +                               &topo_info, &topo_ids);
>>>>     return topo_ids.pkg_id % ms->numa_state->num_nodes;
>>>>  }
>>>>  
>>>> @@ -179,6 +198,9 @@ const CPUArchIdList 
>>>> *x86_possible_cpu_arch_ids(MachineState *ms)
>>>>          return ms->possible_cpus;
>>>>      }
>>>>  
>>>> +    /* Check for apicid encoding */
>>>> +    x86_check_apic_id_encoding(ms);  
>>>
>>> that might crash if user used legacy -numa node,cpus= option
>>> option parser would call reach here before machine->cpu_type is set
>>>
>>> it's better to put this call into x86_cpus_init() like it was done
>>> in previous versions  
>>
>> I have tested with legacy -numa option. It works fine. I can remove this
>> call from here but I need to call the function x86_topo_ids_from_apicid(or
>> x86_topo_ids_from_idx) directly. I cannot use the callback here(because
>> x86_cpus_init has not happened at this point). Is that fine?
> That's fine.
ok. thanks

> x86_topo_ids_from_idx() is the only one that could work correctly early,
> since x86_topo_ids_from_idx_epyc() depends on all numa nodes being already 
> parsed.
> 
> which rises a question do we really need topo_ids_from_idx() callback and 
> x86_topo_ids_from_idx_epyc()?
> I don't see why it's necessary.

yes. We don't need topo_ids_from_idx callback anymore. I will remove it.


> 
>>
>>>   
>>>>      ms->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
>>>>                                    sizeof(CPUArchId) * max_cpus);
>>>>      ms->possible_cpus->len = max_cpus;
>>>> @@ -192,8 +214,8 @@ const CPUArchIdList 
>>>> *x86_possible_cpu_arch_ids(MachineState *ms)
>>>>          ms->possible_cpus->cpus[i].vcpus_count = 1;
>>>>          ms->possible_cpus->cpus[i].arch_id =
>>>>              x86_cpu_apic_id_from_index(x86ms, i);
>>>> -        x86_topo_ids_from_apicid(ms->possible_cpus->cpus[i].arch_id,
>>>> -                                 &topo_info, &topo_ids);
>>>> +        x86ms->topo_ids_from_apicid(ms->possible_cpus->cpus[i].arch_id,
>>>> +                                    &topo_info, &topo_ids);
>>>>          ms->possible_cpus->cpus[i].props.has_socket_id = true;
>>>>          ms->possible_cpus->cpus[i].props.socket_id = topo_ids.pkg_id;
>>>>          if (x86ms->smp_dies > 1) {
>>>> diff --git a/target/i386/cpu.c b/target/i386/cpu.c
>>>> index a3051524a2..19de79d01c 100644
>>>> --- a/target/i386/cpu.c
>>>> +++ b/target/i386/cpu.c
>>>> @@ -1615,6 +1615,10 @@ typedef struct X86CPUDefinition {
>>>>      FeatureWordArray features;
>>>>      const char *model_id;
>>>>      CPUCaches *cache_info;
>>>> +
>>>> +    /* Use AMD EPYC encoding for apic id */
>>>> +    bool use_epyc_apic_id_encoding;
>>>> +
>>>>      /*
>>>>       * Definitions for alternative versions of CPU model.
>>>>       * List is terminated by item with version == 0.
>>>> @@ -1656,6 +1660,13 @@ static const X86CPUVersionDefinition 
>>>> *x86_cpu_def_get_versions(X86CPUDefinition
>>>>      return def->versions ?: default_version_list;
>>>>  }
>>>>  
>>>> +bool cpu_x86_use_epyc_apic_id_encoding(const char *cpu_type)
>>>> +{
>>>> +    X86CPUClass *xcc = X86_CPU_CLASS(object_class_by_name(cpu_type));  
>>>
>>> assert(xcc)  
>>
>> will add and I need to check for cpudef also.
>>
>>>   
>>>> +
>>>> +    return xcc->model->cpudef->use_epyc_apic_id_encoding;
>>>> +}
>>>> +
>>>>  static CPUCaches epyc_cache_info = {
>>>>      .l1d_cache = &(CPUCacheInfo) {
>>>>          .type = DATA_CACHE,
>>>> diff --git a/target/i386/cpu.h b/target/i386/cpu.h
>>>> index 7e9e963d78..6e522fcd34 100644
>>>> --- a/target/i386/cpu.h
>>>> +++ b/target/i386/cpu.h
>>>> @@ -1897,6 +1897,7 @@ void cpu_clear_apic_feature(CPUX86State *env);
>>>>  void host_cpuid(uint32_t function, uint32_t count,
>>>>                  uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t 
>>>> *edx);
>>>>  void host_vendor_fms(char *vendor, int *family, int *model, int 
>>>> *stepping);
>>>> +bool cpu_x86_use_epyc_apic_id_encoding(const char *cpu_type);
>>>>  
>>>>  /* helper.c */
>>>>  bool x86_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
>>>>  
>>>   
>>
>