Messages in this thread Patch in this message | ![/](/images/icornerl.gif) | | From | Rusty Russell <> | Subject | [PATCH 5/4] Make Generic irq_stat Structure Use per-cpu | Date | Thu, 03 Jul 2003 18:47:15 +1000 |
| |
[ Sorry about the counting thing, decided to split this patch after I'd sent the first two. ]
Changes the generic irq_stat structure to use DECLARE_PER_CPU. This saves a little space, and is more efficient on some archs.
Only i386 is fixed here.
Some archs refer to this in assembler: this is easy on UP (append __per_cpu to the symbol name), but more complex on SMP (you need to add __per_cpu_offset[smp_processor_id()]). If that's too hard then you can just stop using the generic structure. Please remember to drop the ____cacheline_aligned from the typedef of irq_cpustat_t!
Name: Make Generic irq_stat Structure Use per-cpu Author: Rusty Russell Status: Tested on 2.5.74 Depends: Percpu/local_softirq_pending.patch.gz
D: Changes the generic irq_stat structure to use DECLARE_PER_CPU. D: This saves a little space, and is more efficient on some archs. D: D: Only i386 is fixed here. D: D: Some archs refer to this in assembler: this is easy on UP (append D: __per_cpu to the symbol name), but more complex on SMP (you need D: to add __per_cpu_offset[smp_processor_id()]). If that's too hard D: then you can just stop using the generic structure. D: Please remember to drop the ____cacheline_aligned from the typedef D: of irq_cpustat_t!
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/apic.c .15631-linux-2.5.74.updated/arch/i386/kernel/apic.c --- .15631-linux-2.5.74/arch/i386/kernel/apic.c 2003-07-03 09:43:41.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/apic.c 2003-07-03 16:24:21.000000000 +1000 @@ -1039,12 +1039,10 @@ inline void smp_local_timer_interrupt(st void smp_apic_timer_interrupt(struct pt_regs regs) { - int cpu = smp_processor_id(); - /* * the NMI deadlock-detector uses this. */ - irq_stat[cpu].apic_timer_irqs++; + __get_cpu_var(irq_stat).apic_timer_irqs++; /* * NOTE! We'd better ACK the irq immediately, diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/io_apic.c .15631-linux-2.5.74.updated/arch/i386/kernel/io_apic.c --- .15631-linux-2.5.74/arch/i386/kernel/io_apic.c 2003-07-03 09:43:41.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/io_apic.c 2003-07-03 16:24:21.000000000 +1000 @@ -304,7 +304,7 @@ struct irq_cpu_info { #define IRQ_DELTA(cpu,irq) (irq_cpu_data[cpu].irq_delta[irq]) #define IDLE_ENOUGH(cpu,now) \ - (idle_cpu(cpu) && ((now) - irq_stat[(cpu)].idle_timestamp > 1)) + (idle_cpu(cpu) && ((now) - per_cpu(irq_stat,(cpu)).idle_timestamp > 1)) #define IRQ_ALLOWED(cpu,allowed_mask) \ ((1 << cpu) & (allowed_mask)) diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/irq.c .15631-linux-2.5.74.updated/arch/i386/kernel/irq.c --- .15631-linux-2.5.74/arch/i386/kernel/irq.c 2003-06-15 11:29:47.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/irq.c 2003-07-03 16:24:21.000000000 +1000 @@ -181,7 +181,8 @@ skip: seq_printf(p, "LOC: "); for (j = 0; j < NR_CPUS; j++) if (cpu_online(j)) - seq_printf(p, "%10u ", irq_stat[j].apic_timer_irqs); + seq_printf(p, "%10u ", + per_cpu(irq_stat, j).apic_timer_irqs); seq_putc(p, '\n'); #endif seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/nmi.c .15631-linux-2.5.74.updated/arch/i386/kernel/nmi.c --- .15631-linux-2.5.74/arch/i386/kernel/nmi.c 2003-06-23 10:52:42.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/nmi.c 2003-07-03 16:24:21.000000000 +1000 @@ -90,7 +90,7 @@ int __init check_nmi_watchdog (void) printk(KERN_INFO "testing NMI watchdog ... "); for (cpu = 0; cpu < NR_CPUS; cpu++) - prev_nmi_count[cpu] = irq_stat[cpu].__nmi_count; + prev_nmi_count[cpu] = nmi_count(cpu); local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks @@ -409,7 +409,7 @@ void nmi_watchdog_tick (struct pt_regs * */ int sum, cpu = smp_processor_id(); - sum = irq_stat[cpu].apic_timer_irqs; + sum = __get_cpu_var(irq_stat).apic_timer_irqs; if (last_irq_sums[cpu] == sum) { /* diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/process.c .15631-linux-2.5.74.updated/arch/i386/kernel/process.c --- .15631-linux-2.5.74/arch/i386/kernel/process.c 2003-06-23 10:52:42.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/process.c 2003-07-03 16:24:21.000000000 +1000 @@ -141,7 +141,7 @@ void cpu_idle (void) void (*idle)(void) = pm_idle; if (!idle) idle = default_idle; - irq_stat[smp_processor_id()].idle_timestamp = jiffies; + __get_cpu_var(irq_stat).idle_timestamp = jiffies; while (!need_resched()) idle(); schedule(); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/i386/kernel/traps.c .15631-linux-2.5.74.updated/arch/i386/kernel/traps.c --- .15631-linux-2.5.74/arch/i386/kernel/traps.c 2003-07-03 09:43:41.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/i386/kernel/traps.c 2003-07-03 16:24:21.000000000 +1000 @@ -475,7 +475,7 @@ asmlinkage void do_nmi(struct pt_regs * nmi_enter(); cpu = smp_processor_id(); - ++nmi_count(cpu); + ++local_nmi_count(); if (!nmi_callback(regs, cpu)) default_do_nmi(regs); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/x86_64/kernel/irq.c .15631-linux-2.5.74.updated/arch/x86_64/kernel/irq.c --- .15631-linux-2.5.74/arch/x86_64/kernel/irq.c 2003-06-15 11:29:50.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/x86_64/kernel/irq.c 2003-07-03 16:24:22.000000000 +1000 @@ -173,7 +173,7 @@ skip: seq_printf(p, "NMI: "); for (j = 0; j < NR_CPUS; j++) if (cpu_online(j)) - seq_printf(p, "%10u ", cpu_pda[j].__nmi_count); + seq_printf(p, "%10u ", nmi_count(j)); seq_putc(p, '\n'); #ifdef CONFIG_X86_LOCAL_APIC seq_printf(p, "LOC: "); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/arch/x86_64/kernel/nmi.c .15631-linux-2.5.74.updated/arch/x86_64/kernel/nmi.c --- .15631-linux-2.5.74/arch/x86_64/kernel/nmi.c 2003-07-03 09:43:46.000000000 +1000 +++ .15631-linux-2.5.74.updated/arch/x86_64/kernel/nmi.c 2003-07-03 16:26:24.000000000 +1000 @@ -86,17 +86,17 @@ int __init check_nmi_watchdog (void) printk(KERN_INFO "testing NMI watchdog ... "); for (cpu = 0; cpu < NR_CPUS; cpu++) - counts[cpu] = cpu_pda[cpu].__nmi_count; + counts[cpu] = nmi_count(cpu); local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks for (cpu = 0; cpu < NR_CPUS; cpu++) { if (!cpu_online(cpu)) continue; - if (cpu_pda[cpu].__nmi_count - counts[cpu] <= 5) { + if (nmi_count(cpu) - counts[cpu] <= 5) { printk("CPU#%d: NMI appears to be stuck (%d)!\n", cpu, - cpu_pda[cpu].__nmi_count); + nmi_count(cpu)); nmi_active = 0; return -1; } diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/include/asm-i386/hardirq.h .15631-linux-2.5.74.updated/include/asm-i386/hardirq.h --- .15631-linux-2.5.74/include/asm-i386/hardirq.h 2003-07-03 16:24:19.000000000 +1000 +++ .15631-linux-2.5.74.updated/include/asm-i386/hardirq.h 2003-07-03 16:24:22.000000000 +1000 @@ -10,7 +10,7 @@ typedef struct { unsigned long idle_timestamp; unsigned int __nmi_count; /* arch dependent */ unsigned int apic_timer_irqs; /* arch dependent */ -} ____cacheline_aligned irq_cpustat_t; +} irq_cpustat_t; #include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/include/linux/irq_cpustat.h .15631-linux-2.5.74.updated/include/linux/irq_cpustat.h --- .15631-linux-2.5.74/include/linux/irq_cpustat.h 2003-07-03 16:24:19.000000000 +1000 +++ .15631-linux-2.5.74.updated/include/linux/irq_cpustat.h 2003-07-03 16:24:22.000000000 +1000 @@ -10,6 +10,7 @@ */ #include <linux/config.h> +#include <linux/percpu.h> /* * Simple wrappers reducing source bloat. Define all irq_stat fields @@ -18,19 +19,17 @@ */ #ifndef __ARCH_IRQ_STAT -extern irq_cpustat_t irq_stat[]; /* defined in asm/hardirq.h */ -#ifdef CONFIG_SMP -#define __IRQ_STAT(cpu, member) (irq_stat[cpu].member) -#else -#define __IRQ_STAT(cpu, member) ((void)(cpu), irq_stat[0].member) -#endif +DECLARE_PER_CPU(irq_cpustat_t, irq_stat); /* defined in asm/hardirq.h */ +#define __IRQ_STAT(cpu, member) (per_cpu(irq_stat, (cpu)).member) +#define __LOCAL_IRQ_STAT(member) (__get_cpu_var(irq_stat).member) #endif /* arch independent irq_stat fields */ -#define softirq_pending(cpu) __IRQ_STAT((cpu), __softirq_pending) -#define local_softirq_pending() softirq_pending(smp_processor_id()) +#define local_softirq_pending() __LOCAL_IRQ_STAT(__softirq_pending) /* arch dependent irq_stat fields */ -#define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) /* i386 */ +/* i386, x86-64 */ +#define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) +#define local_nmi_count() __LOCAL_IRQ_STAT(__nmi_count) #endif /* __irq_cpustat_h */ diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .15631-linux-2.5.74/kernel/softirq.c .15631-linux-2.5.74.updated/kernel/softirq.c --- .15631-linux-2.5.74/kernel/softirq.c 2003-07-03 16:24:19.000000000 +1000 +++ .15631-linux-2.5.74.updated/kernel/softirq.c 2003-07-03 16:24:22.000000000 +1000 @@ -36,8 +36,8 @@ */ #ifndef __ARCH_IRQ_STAT -irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; -EXPORT_SYMBOL(irq_stat); +DEFINE_PER_CPU(irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL(irq_stat); #endif static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp; -- Anyone who quotes me in their sig is an idiot. -- Rusty Russell. - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
| ![\](/images/icornerr.gif) |