Messages in this thread Patch in this message | | | From | Wyes Karny <> | Subject | [RFC] amd_uncore/rapl: Add per-core energy tracking support for AMD | Date | Mon, 11 Sep 2023 17:46:52 +0000 |
| |
Introduce distinct energy monitoring event in the amd_uncore driver to accommodate per-core power consumption readings. Unlike the existing energy-cores event in the power PMU, which aggregates power data per socket/package [1], this new event aligns with AMD's per-core RAPL counters. This change caters to the need for separate power consumption measurements for individual cores.
Running a workload pinned to CPU 1 give below output with perf stat:
$sudo perf stat --per-core -C 0-7 -I 1000 -e amd_rapl/energy-cores/
time core cpus counts unit events 17.176191314 S0-D0-C0 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C1 1 4.35 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C2 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C3 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C4 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C5 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C6 1 0.02 Joules amd_rapl/energy-cores/ 17.176191314 S0-D0-C7 1 0.02 Joules amd_rapl/energy-cores/
[1]: https://lore.kernel.org/lkml/3e766f0e-37d4-0f82-3868-31b14228868d@linux.intel.com/
Signed-off-by: Wyes Karny <wyes.karny@amd.com> --- arch/x86/events/amd/uncore.c | 213 +++++++++++++++++++++++++++++++++-- 1 file changed, 202 insertions(+), 11 deletions(-)
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 83f15fe411b3..e03b12e20ebd 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -26,6 +26,7 @@ #define RDPMC_BASE_LLC 10 #define COUNTER_SHIFT 16 +#define RAPL_CNTR_WIDTH 32 #undef pr_fmt #define pr_fmt(fmt) "amd_uncore: " fmt @@ -35,6 +36,8 @@ static int num_counters_llc; static int num_counters_nb; static bool l3_mask; +static int rapl_hw_unit __read_mostly; + static HLIST_HEAD(uncore_unused_list); struct amd_uncore { @@ -52,12 +55,15 @@ struct amd_uncore { static struct amd_uncore * __percpu *amd_uncore_nb; static struct amd_uncore * __percpu *amd_uncore_llc; +static struct amd_uncore * __percpu *amd_uncore_rapl; static struct pmu amd_nb_pmu; static struct pmu amd_llc_pmu; +static struct pmu amd_rapl_pmu; static cpumask_t amd_nb_active_mask; static cpumask_t amd_llc_active_mask; +static cpumask_t amd_rapl_active_mask; static bool is_nb_event(struct perf_event *event) { @@ -69,44 +75,78 @@ static bool is_llc_event(struct perf_event *event) return event->pmu->type == amd_llc_pmu.type; } +static bool is_rapl_event(struct perf_event *event) +{ + return event->pmu->type == amd_rapl_pmu.type; +} + static struct amd_uncore *event_to_amd_uncore(struct perf_event *event) { if (is_nb_event(event) && amd_uncore_nb) return *per_cpu_ptr(amd_uncore_nb, event->cpu); else if (is_llc_event(event) && amd_uncore_llc) return *per_cpu_ptr(amd_uncore_llc, event->cpu); + else if (is_rapl_event(event) && amd_uncore_rapl) + return *per_cpu_ptr(amd_uncore_rapl, event->cpu); return NULL; } +static inline u64 rapl_scale(u64 v) +{ + return v << (32 - rapl_hw_unit); +} + static void amd_uncore_read(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; u64 prev, new; s64 delta; + int shift; /* * since we do not enable counter overflow interrupts, * we do not have to worry about prev_count changing on us */ - prev = local64_read(&hwc->prev_count); - rdpmcl(hwc->event_base_rdpmc, new); - local64_set(&hwc->prev_count, new); - delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); - delta >>= COUNTER_SHIFT; + if (is_rapl_event(event)) { + shift = RAPL_CNTR_WIDTH; +again: + prev = local64_read(&hwc->prev_count); + rdmsrl(hwc->event_base, new); + if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) { + cpu_relax(); + goto again; + } + delta = (new << shift) - (prev << shift); + delta >>= shift; + delta = rapl_scale(delta); + } else { + prev = local64_read(&hwc->prev_count); + rdpmcl(hwc->event_base_rdpmc, new); + local64_set(&hwc->prev_count, new); + delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); + delta >>= COUNTER_SHIFT; + } local64_add(delta, &event->count); } static void amd_uncore_start(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; + u64 new; if (flags & PERF_EF_RELOAD) wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); + if (is_rapl_event(event)) { + rdmsrl(hwc->event_base, new); + local64_set(&hwc->prev_count, new); + } + hwc->state = 0; - wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); + if (hwc->config_base) + wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); perf_event_update_userpage(event); } @@ -114,7 +154,8 @@ static void amd_uncore_stop(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config); + if (hwc->config_base) + wrmsrl(hwc->config_base, hwc->config); hwc->state |= PERF_HES_STOPPED; if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { @@ -153,9 +194,15 @@ static int amd_uncore_add(struct perf_event *event, int flags) if (hwc->idx == -1) return -EBUSY; - hwc->config_base = uncore->msr_base + (2 * hwc->idx); - hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); - hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; + if (is_rapl_event(event)) { + hwc->config_base = 0; + hwc->event_base = uncore->msr_base; + hwc->event_base_rdpmc = 0; + } else { + hwc->config_base = uncore->msr_base + (2 * hwc->idx); + hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); + hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; + } hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; /* @@ -285,6 +332,8 @@ static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, active_mask = &amd_nb_active_mask; else if (pmu->type == amd_llc_pmu.type) active_mask = &amd_llc_active_mask; + else if (pmu->type == amd_rapl_pmu.type) + active_mask = &amd_rapl_active_mask; else return 0; @@ -326,6 +375,69 @@ DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ +#define RAPL_EVENT_ATTR_STR(_name, v, str) \ +static struct perf_pmu_events_attr event_attr_##v = { \ + .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \ + .id = 0, \ + .event_str = str, \ +}; + +RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01"); +RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules"); +RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_cores_scale, "2.3283064365386962890625e-10"); + +/* + * There are no default events, but we need to create + * "events" group (with empty attrs) before updating + * it with detected events. + */ +static struct attribute *attrs_empty[] = { + NULL, +}; + +static struct attribute_group rapl_pmu_events_group = { + .name = "events", + .attrs = attrs_empty, +}; + +PMU_FORMAT_ATTR(event, "config:0-7"); +static struct attribute *rapl_formats_attr[] = { + &format_attr_event.attr, + NULL, +}; + +static struct attribute_group rapl_pmu_format_group = { + .name = "format", + .attrs = rapl_formats_attr, +}; + +static const struct attribute_group *rapl_attr_groups[] = { + &amd_uncore_attr_group, + &rapl_pmu_format_group, + &rapl_pmu_events_group, + NULL, +}; + +static struct attribute *rapl_events_cores[] = { + &event_attr_rapl_cores.attr.attr, + &event_attr_rapl_cores_unit.attr.attr, + &event_attr_rapl_cores_scale.attr.attr, + NULL, +}; + +static struct attribute_group rapl_events_cores_group = { + .name = "events", + .attrs = rapl_events_cores, +}; + + +static const struct attribute_group *rapl_attr_update[] = { + &rapl_events_cores_group, + NULL, +}; + + + /* Common DF and NB attributes */ static struct attribute *amd_uncore_df_format_attr[] = { &format_attr_event12.attr, /* event */ @@ -425,6 +537,21 @@ static struct pmu amd_llc_pmu = { .module = THIS_MODULE, }; +static struct pmu amd_rapl_pmu = { + .task_ctx_nr = perf_invalid_context, + .attr_groups = rapl_attr_groups, + .attr_update = rapl_attr_update, + .name = "amd_rapl", + .event_init = amd_uncore_event_init, + .add = amd_uncore_add, + .del = amd_uncore_del, + .start = amd_uncore_start, + .stop = amd_uncore_stop, + .read = amd_uncore_read, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, + .module = THIS_MODULE, +}; + static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) { return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, @@ -440,7 +567,7 @@ amd_uncore_events_alloc(unsigned int num, unsigned int cpu) static int amd_uncore_cpu_up_prepare(unsigned int cpu) { - struct amd_uncore *uncore_nb = NULL, *uncore_llc = NULL; + struct amd_uncore *uncore_nb = NULL, *uncore_llc = NULL, *uncore_rapl = NULL; if (amd_uncore_nb) { *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; @@ -478,6 +605,23 @@ static int amd_uncore_cpu_up_prepare(unsigned int cpu) *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc; } + if (amd_uncore_rapl) { + *per_cpu_ptr(amd_uncore_rapl, cpu) = NULL; + uncore_rapl = amd_uncore_alloc(cpu); + if (!uncore_rapl) + goto fail; + uncore_rapl->cpu = cpu; + uncore_rapl->num_counters = 1; + uncore_rapl->msr_base = MSR_AMD_CORE_ENERGY_STATUS; + uncore_rapl->active_mask = &amd_rapl_active_mask; + uncore_rapl->pmu = &amd_rapl_pmu; + uncore_rapl->events = amd_uncore_events_alloc(1, cpu); + if (!uncore_nb->events) + goto fail; + uncore_rapl->id = -1; + *per_cpu_ptr(amd_uncore_rapl, cpu) = uncore_rapl; + } + return 0; fail: @@ -543,6 +687,14 @@ static int amd_uncore_cpu_starting(unsigned int cpu) *per_cpu_ptr(amd_uncore_llc, cpu) = uncore; } + if (amd_uncore_rapl) { + uncore = *per_cpu_ptr(amd_uncore_rapl, cpu); + uncore->id = topology_core_id(cpu); + + uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_rapl); + *per_cpu_ptr(amd_uncore_rapl, cpu) = uncore; + } + return 0; } @@ -577,6 +729,9 @@ static int amd_uncore_cpu_online(unsigned int cpu) if (amd_uncore_llc) uncore_online(cpu, amd_uncore_llc); + if (amd_uncore_rapl) + uncore_online(cpu, amd_uncore_rapl); + return 0; } @@ -614,6 +769,9 @@ static int amd_uncore_cpu_down_prepare(unsigned int cpu) if (amd_uncore_llc) uncore_down_prepare(cpu, amd_uncore_llc); + if (amd_uncore_rapl) + uncore_down_prepare(cpu, amd_uncore_rapl); + return 0; } @@ -640,6 +798,9 @@ static int amd_uncore_cpu_dead(unsigned int cpu) if (amd_uncore_llc) uncore_dead(cpu, amd_uncore_llc); + if (amd_uncore_rapl) + uncore_dead(cpu, amd_uncore_rapl); + return 0; } @@ -649,6 +810,7 @@ static int __init amd_uncore_init(void) struct attribute **l3_attr = amd_uncore_l3_format_attr; union cpuid_0x80000022_ebx ebx; int ret = -ENODEV; + u64 msr_rapl_power_unit_bits; if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) @@ -730,6 +892,29 @@ static int __init amd_uncore_init(void) ret = 0; } + if (boot_cpu_has(X86_FEATURE_RAPL)) { + if (boot_cpu_data.x86 >= 0x19) { + amd_uncore_rapl = alloc_percpu(struct amd_uncore *); + if (!amd_uncore_rapl) { + ret = -ENOMEM; + goto fail_llc; + } + + ret = perf_pmu_register(&amd_rapl_pmu, amd_rapl_pmu.name, -1); + if (ret) + goto fail_llc; + pr_info("%s PMU detected\n", amd_rapl_pmu.name); + + if (rdmsrl_safe(MSR_AMD_RAPL_POWER_UNIT, &msr_rapl_power_unit_bits)) { + pr_err("RAPL POWER unit reg read error\n"); + ret = -1; + } else { + rapl_hw_unit = (msr_rapl_power_unit_bits >> 8) & 0x1FULL; + ret = 0; + } + } + } + /* * Install callbacks. Core will call them for each online cpu. */ @@ -780,6 +965,12 @@ static void __exit amd_uncore_exit(void) free_percpu(amd_uncore_nb); amd_uncore_nb = NULL; } + + if (boot_cpu_has(X86_FEATURE_RAPL)) { + perf_pmu_unregister(&amd_rapl_pmu); + free_percpu(amd_uncore_rapl); + amd_uncore_rapl = NULL; + } } module_init(amd_uncore_init); -- 2.34.1
| |