Messages in this thread | | | From | Len Brown <> | Date | Thu, 13 Aug 2020 17:50:54 -0400 | Subject | Re: [PATCH 2/2][RFC] tools/power turbostat: Introduce reliable RAPL display |
| |
why not simply use nanosleep(2)
On Tue, Mar 31, 2020 at 2:45 PM Chen Yu <yu.c.chen@intel.com> wrote: > > Since the RAPL Joule Counter is 32 bit, turbostat would > only print a *star* instead of printing the actual energy > consumed to indicate the overflow due to long duration. > This does not meet the requirement on servers as the > sampling time of turbostat is usually very long on servers. > > So maintain a set of MSR buffer, and updates them > periodically before the 32bit msr register wrapped round. > > The idea is similar to the implementation of ktime_get(): > get_msr_sum() is used rather than get_msr() to get the > accumulated MSR. > > This can be illustrated below: > > MSR timer: > total_rapl_msr += (current_rapl_msr - last_rapl_msr); > > get_msr_sum(): > return (current_rapl_msr - last_rapl_msr) + total_rapl_msr; > > Originally-by: Aaron Lu <aaron.lwe@gmail.com> > Signed-off-by: Chen Yu <yu.c.chen@intel.com> > --- > tools/power/x86/turbostat/Makefile | 2 +- > tools/power/x86/turbostat/turbostat.c | 292 ++++++++++++++++++++++++-- > 2 files changed, 274 insertions(+), 20 deletions(-) > > diff --git a/tools/power/x86/turbostat/Makefile b/tools/power/x86/turbostat/Makefile > index 2b6551269e43..d08765531bcb 100644 > --- a/tools/power/x86/turbostat/Makefile > +++ b/tools/power/x86/turbostat/Makefile > @@ -16,7 +16,7 @@ override CFLAGS += -D_FORTIFY_SOURCE=2 > > %: %.c > @mkdir -p $(BUILD_OUTPUT) > - $(CC) $(CFLAGS) $< -o $(BUILD_OUTPUT)/$@ $(LDFLAGS) -lcap > + $(CC) $(CFLAGS) $< -o $(BUILD_OUTPUT)/$@ $(LDFLAGS) -lcap -lrt > > .PHONY : clean > clean : > diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c > index 95f3047e94ae..a8979bec97e4 100644 > --- a/tools/power/x86/turbostat/turbostat.c > +++ b/tools/power/x86/turbostat/turbostat.c > @@ -47,6 +47,7 @@ unsigned int sums_need_wide_columns; > unsigned int rapl_joules; > unsigned int summary_only; > unsigned int list_header_only; > +unsigned int longtime; > unsigned int dump_only; > unsigned int do_snb_cstates; > unsigned int do_knl_cstates; > @@ -259,6 +260,113 @@ struct msr_counter { > #define SYSFS_PERCPU (1 << 1) > }; > > +/* > + * The accumulated sum of MSR is defined as a monotonic > + * increasing MSR, it will be accumulated periodically, > + * despite its register's bit width. > + */ > +enum { > + IDX_PKG_ENERGY, > + IDX_DRAM_ENERGY, > + IDX_PP0_ENERGY, > + IDX_PP1_ENERGY, > + IDX_PKG_PERF, > + IDX_DRAM_PERF, > + IDX_COUNT, > +}; > + > +int get_msr_sum(int cpu, off_t offset, unsigned long long *msr); > + > +struct msr_sum_array { > + /* get_msr_sum() = sum + (get_msr() - last) */ > + struct { > + /*The accumulated MSR value is updated by the timer*/ > + unsigned long long sum; > + /*The MSR footprint recorded in last timer*/ > + unsigned long long last; > + } entries[IDX_COUNT]; > +}; > + > +/* The percpu MSR sum array.*/ > +struct msr_sum_array *per_cpu_msr_sum; > + > +int idx_to_offset(int idx) > +{ > + int offset; > + > + switch (idx) { > + case IDX_PKG_ENERGY: > + offset = MSR_PKG_ENERGY_STATUS; > + break; > + case IDX_DRAM_ENERGY: > + offset = MSR_DRAM_ENERGY_STATUS; > + break; > + case IDX_PP0_ENERGY: > + offset = MSR_PP0_ENERGY_STATUS; > + break; > + case IDX_PP1_ENERGY: > + offset = MSR_PP1_ENERGY_STATUS; > + break; > + case IDX_PKG_PERF: > + offset = MSR_PKG_PERF_STATUS; > + break; > + case IDX_DRAM_PERF: > + offset = MSR_DRAM_PERF_STATUS; > + break; > + default: > + offset = -1; > + } > + return offset; > +} > + > +int offset_to_idx(int offset) > +{ > + int idx; > + > + switch (offset) { > + case MSR_PKG_ENERGY_STATUS: > + idx = IDX_PKG_ENERGY; > + break; > + case MSR_DRAM_ENERGY_STATUS: > + idx = IDX_DRAM_ENERGY; > + break; > + case MSR_PP0_ENERGY_STATUS: > + idx = IDX_PP0_ENERGY; > + break; > + case MSR_PP1_ENERGY_STATUS: > + idx = IDX_PP1_ENERGY; > + break; > + case MSR_PKG_PERF_STATUS: > + idx = IDX_PKG_PERF; > + break; > + case MSR_DRAM_PERF_STATUS: > + idx = IDX_DRAM_PERF; > + break; > + default: > + idx = -1; > + } > + return idx; > +} > + > +int idx_valid(int idx) > +{ > + switch (idx) { > + case IDX_PKG_ENERGY: > + return do_rapl & RAPL_PKG; > + case IDX_DRAM_ENERGY: > + return do_rapl & RAPL_DRAM; > + case IDX_PP0_ENERGY: > + return do_rapl & RAPL_CORES_ENERGY_STATUS; > + case IDX_PP1_ENERGY: > + return do_rapl & RAPL_GFX; > + case IDX_PKG_PERF: > + return do_rapl & RAPL_PKG_PERF_STATUS; > + case IDX_DRAM_PERF: > + return do_rapl & RAPL_DRAM_PERF_STATUS; > + default: > + return 0; > + } > +} > struct sys_counters { > unsigned int added_thread_counters; > unsigned int added_core_counters; > @@ -551,6 +659,7 @@ void help(void) > " Override default 5-second measurement interval\n" > " -J, --Joules displays energy in Joules instead of Watts\n" > " -l, --list list column headers only\n" > + " -L, --Longtime long time duration support\n" > " -n, --num_iterations num\n" > " number of the measurement iterations\n" > " -o, --out file\n" > @@ -1962,34 +2071,70 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) > p->sys_lpi = cpuidle_cur_sys_lpi_us; > > if (do_rapl & RAPL_PKG) { > - if (get_msr(cpu, MSR_PKG_ENERGY_STATUS, &msr)) > - return -13; > - p->energy_pkg = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_PKG_ENERGY_STATUS, &msr)) > + return -13; > + p->energy_pkg = msr; > + } else { > + if (get_msr(cpu, MSR_PKG_ENERGY_STATUS, &msr)) > + return -13; > + p->energy_pkg = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_CORES_ENERGY_STATUS) { > - if (get_msr(cpu, MSR_PP0_ENERGY_STATUS, &msr)) > - return -14; > - p->energy_cores = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_PP0_ENERGY_STATUS, &msr)) > + return -14; > + p->energy_cores = msr; > + } else { > + if (get_msr(cpu, MSR_PP0_ENERGY_STATUS, &msr)) > + return -14; > + p->energy_cores = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_DRAM) { > - if (get_msr(cpu, MSR_DRAM_ENERGY_STATUS, &msr)) > - return -15; > - p->energy_dram = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_DRAM_ENERGY_STATUS, &msr)) > + return -15; > + p->energy_dram = msr; > + } else { > + if (get_msr(cpu, MSR_DRAM_ENERGY_STATUS, &msr)) > + return -15; > + p->energy_dram = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_GFX) { > - if (get_msr(cpu, MSR_PP1_ENERGY_STATUS, &msr)) > - return -16; > - p->energy_gfx = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_PP1_ENERGY_STATUS, &msr)) > + return -16; > + p->energy_gfx = msr; > + } else { > + if (get_msr(cpu, MSR_PP1_ENERGY_STATUS, &msr)) > + return -16; > + p->energy_gfx = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_PKG_PERF_STATUS) { > - if (get_msr(cpu, MSR_PKG_PERF_STATUS, &msr)) > - return -16; > - p->rapl_pkg_perf_status = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_DRAM_PERF_STATUS, &msr)) > + return -16; > + p->rapl_dram_perf_status = msr; > + } else { > + if (get_msr(cpu, MSR_PKG_PERF_STATUS, &msr)) > + return -16; > + p->rapl_pkg_perf_status = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_DRAM_PERF_STATUS) { > - if (get_msr(cpu, MSR_DRAM_PERF_STATUS, &msr)) > - return -16; > - p->rapl_dram_perf_status = msr & 0xFFFFFFFF; > + if (longtime) { > + if (get_msr_sum(cpu, MSR_DRAM_PERF_STATUS, &msr)) > + return -16; > + p->rapl_dram_perf_status = msr; > + } else { > + if (get_msr(cpu, MSR_DRAM_PERF_STATUS, &msr)) > + return -16; > + p->rapl_dram_perf_status = msr & 0xFFFFFFFF; > + } > } > if (do_rapl & RAPL_AMD_F17H) { > if (get_msr(cpu, MSR_PKG_ENERGY_STAT, &msr)) > @@ -3053,6 +3198,109 @@ void do_sleep(void) > } > } > > +int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) > +{ > + int ret, idx; > + unsigned long long msr_cur, msr_last; > + > + if (!per_cpu_msr_sum) > + return 1; > + > + idx = offset_to_idx(offset); > + if (idx < 0) > + return idx; > + /* get_msr_sum() = sum + (get_msr() - last) */ > + ret = get_msr(cpu, offset, &msr_cur); > + if (ret) > + return ret; > + msr_last = per_cpu_msr_sum[cpu].entries[idx].last; > + DELTA_WRAP32(msr_cur, msr_last); > + *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum; > + > + return 0; > +} > + > +timer_t timerid; > + > +/* Timer callback, update the sum of MSRs periodically. */ > +static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg_data *p) > +{ > + int i, ret; > + int cpu = t->cpu_id; > + > + for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) { > + unsigned long long msr_cur, msr_last; > + int offset; > + > + if (!idx_valid(i)) > + continue; > + offset = idx_to_offset(i); > + if (offset < 0) > + continue; > + ret = get_msr(cpu, offset, &msr_cur); > + if (ret) { > + fprintf(outf, "Can not update msr(0x%x)\n", offset); > + continue; > + } > + > + msr_last = per_cpu_msr_sum[cpu].entries[i].last; > + per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff; > + > + DELTA_WRAP32(msr_cur, msr_last); > + per_cpu_msr_sum[cpu].entries[i].sum += msr_last; > + } > + return 0; > +} > + > +static void > +msr_record_handler(union sigval v) > +{ > + for_all_cpus(update_msr_sum, EVEN_COUNTERS); > +} > + > +void msr_longtime_record(void) > +{ > + struct itimerspec its; > + struct sigevent sev; > + > + per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array)); > + if (!per_cpu_msr_sum) { > + fprintf(outf, "Can not allocate memory for long time MSR.\n"); > + return; > + } > + /* > + * Signal handler might be restricted, so use thread notifier instead. > + */ > + memset(&sev, 0, sizeof(struct sigevent)); > + sev.sigev_notify = SIGEV_THREAD; > + sev.sigev_notify_function = msr_record_handler; > + > + sev.sigev_value.sival_ptr = &timerid; > + if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) { > + fprintf(outf, "Can not create timer.\n"); > + goto release_msr; > + } > + > + its.it_value.tv_sec = 0; > + its.it_value.tv_nsec = 1; > + /* > + * A wraparound time of around 60 secs when power consumption > + * is high, use 50 secs. > + */ > + its.it_interval.tv_sec = 50; > + its.it_interval.tv_nsec = 0; > + > + if (timer_settime(timerid, 0, &its, NULL) == -1) { > + fprintf(outf, "Can not set timer.\n"); > + goto release_timer; > + } > + return; > + > + release_timer: > + timer_delete(timerid); > + release_msr: > + free(per_cpu_msr_sum); > +} > > void turbostat_loop() > { > @@ -5735,6 +5983,7 @@ void cmdline(int argc, char **argv) > {"hide", required_argument, 0, 'H'}, // meh, -h taken by --help > {"Joules", no_argument, 0, 'J'}, > {"list", no_argument, 0, 'l'}, > + {"Longtime", no_argument, 0, 'L'}, > {"out", required_argument, 0, 'o'}, > {"quiet", no_argument, 0, 'q'}, > {"show", required_argument, 0, 's'}, > @@ -5746,7 +5995,7 @@ void cmdline(int argc, char **argv) > > progname = argv[0]; > > - while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qST:v", > + while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:JLn:o:qST:v", > long_options, &option_index)) != -1) { > switch (opt) { > case 'a': > @@ -5800,6 +6049,9 @@ void cmdline(int argc, char **argv) > list_header_only++; > quiet++; > break; > + case 'L': > + longtime = 1; > + break; > case 'o': > outf = fopen_or_die(optarg, "w"); > break; > @@ -5864,6 +6116,8 @@ int main(int argc, char **argv) > return 0; > } > > + if (longtime) > + msr_longtime_record(); > /* > * if any params left, it must be a command to fork > */ > -- > 2.17.1 >
-- Len Brown, Intel Open Source Technology Center
| |