Messages in this thread Patch in this message | | | From | Xu Lu <> | Subject | [RFC 02/12] riscv: Make CSR_IE register part of context | Date | Mon, 23 Oct 2023 16:29:01 +0800 |
| |
This commit makes CSR_IE register part of thread context.
Kernel nowadays saves and restores irq status of each thread via CSR_STATUS register. When a thread traps into kernel, irq status of it is automatically stored in SR_PIE field of CSR_STATUS by hardware. And when kernel returns back, irq status will be automatically restored from CSR_STATUS.
Things get different when we switch to CSR_IE masking for irq disabling. Hardware won't save or restore CSR_IE value during traps. In this case, when trapped into kernel, we should save CSR_IE value for previous thread manually and then clear all CSR_IE bits to disable irqs during traps. Also, we should manually restore SIE field of CSR_STATUS as we do not depends on it to disable irqs. When kernel returns back, we manually restore CSR_IE value from previous saved value.
Signed-off-by: Xu Lu <luxu.kernel@bytedance.com> Signed-off-by: Hangjing Li <lihangjing@bytedance.com> Reviewed-by: Liang Deng <dengliang.1214@bytedance.com> Reviewed-by: Yu Li <liyu.yukiteru@bytedance.com> --- arch/riscv/include/asm/csr.h | 17 +++++++++++++++++ arch/riscv/include/asm/ptrace.h | 3 +++ arch/riscv/kernel/asm-offsets.c | 3 +++ arch/riscv/kernel/entry.S | 13 +++++++++++++ arch/riscv/kernel/process.c | 6 ++++++ arch/riscv/kernel/suspend_entry.S | 1 + drivers/clocksource/timer-clint.c | 4 ++++ drivers/clocksource/timer-riscv.c | 4 ++++ drivers/irqchip/irq-riscv-intc.c | 4 ++++ drivers/perf/riscv_pmu_sbi.c | 4 ++++ 10 files changed, 59 insertions(+)
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 777cb8299551..6520bd826d52 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -7,6 +7,7 @@ #define _ASM_RISCV_CSR_H #include <asm/asm.h> +#include <asm/hwcap.h> #include <linux/bits.h> /* Status register flags */ @@ -451,6 +452,22 @@ #define IE_TIE (_AC(0x1, UL) << RV_IRQ_TIMER) #define IE_EIE (_AC(0x1, UL) << RV_IRQ_EXT) +#ifdef CONFIG_RISCV_PSEUDO_NMI +#define IRQS_ENABLED_IE (IE_SIE | IE_TIE | IE_EIE) +#define irqs_enabled_ie \ +({ \ + unsigned long __v; \ + asm (ALTERNATIVE( \ + "li %0, " __stringify(IRQS_ENABLED_IE) "\n\t" \ + "nop", \ + "li %0, " __stringify(IRQS_ENABLED_IE | SIP_LCOFIP),\ + 0, RISCV_ISA_EXT_SSCOFPMF, \ + CONFIG_RISCV_PSEUDO_NMI) \ + : "=r"(__v) : : ); \ + __v; \ +}) +#endif /* CONFIG_RISCV_PSEUDO_NMI */ + #ifndef __ASSEMBLY__ #define csr_swap(csr, val) \ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index b5b0adcc85c1..b57d3a6b232f 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -47,6 +47,9 @@ struct pt_regs { unsigned long t6; /* Supervisor/Machine CSRs */ unsigned long status; +#ifdef CONFIG_RISCV_PSEUDO_NMI + unsigned long ie; +#endif unsigned long badaddr; unsigned long cause; /* a0 value before the syscall */ diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index d6a75aac1d27..165f6f9fc458 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -112,6 +112,9 @@ void asm_offsets(void) OFFSET(PT_GP, pt_regs, gp); OFFSET(PT_ORIG_A0, pt_regs, orig_a0); OFFSET(PT_STATUS, pt_regs, status); +#ifdef CONFIG_RISCV_PSEUDO_NMI + OFFSET(PT_IE, pt_regs, ie); +#endif OFFSET(PT_BADADDR, pt_regs, badaddr); OFFSET(PT_CAUSE, pt_regs, cause); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 143a2bb3e697..19ba7c4520b9 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -65,6 +65,10 @@ _save_context: REG_S s3, PT_BADADDR(sp) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) +#ifdef CONFIG_RISCV_PSEUDO_NMI + csrr s0, CSR_IE + REG_S s0, PT_IE(sp) +#endif /* CONFIG_RISCV_PSEUDO_NMI */ /* * Set the scratch register to 0, so that if a recursive exception @@ -153,6 +157,11 @@ SYM_CODE_START_NOALIGN(ret_from_exception) csrw CSR_STATUS, a0 csrw CSR_EPC, a2 +#ifdef CONFIG_RISCV_PSEUDO_NMI + REG_L s0, PT_IE(sp) + csrw CSR_IE, s0 +#endif /* CONFIG_RISCV_PSEUDO_NMI */ + REG_L x1, PT_RA(sp) REG_L x3, PT_GP(sp) REG_L x4, PT_TP(sp) @@ -251,6 +260,10 @@ restore_caller_reg: REG_S s3, PT_BADADDR(sp) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) +#ifdef CONFIG_RISCV_PSEUDO_NMI + csrr s0, CSR_IE + REG_S s0, PT_IE(sp) +#endif /* CONFIG_RISCV_PSEUDO_NMI */ move a0, sp tail handle_bad_stack SYM_CODE_END(handle_kernel_stack_overflow) diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index e32d737e039f..9663bae23c57 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -115,6 +115,9 @@ void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { regs->status = SR_PIE; +#ifdef CONFIG_RISCV_PSEUDO_NMI + regs->ie = irqs_enabled_ie; +#endif if (has_fpu()) { regs->status |= SR_FS_INITIAL; /* @@ -189,6 +192,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) childregs->gp = gp_in_global; /* Supervisor/Machine, irqs on: */ childregs->status = SR_PP | SR_PIE; +#ifdef CONFIG_RISCV_PSEUDO_NMI + childregs->ie = irqs_enabled_ie; +#endif p->thread.s[0] = (unsigned long)args->fn; p->thread.s[1] = (unsigned long)args->fn_arg; diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S index f7960c7c5f9e..6825f4836be4 100644 --- a/arch/riscv/kernel/suspend_entry.S +++ b/arch/riscv/kernel/suspend_entry.S @@ -47,6 +47,7 @@ ENTRY(__cpu_suspend_enter) REG_S t0, (SUSPEND_CONTEXT_REGS + PT_EPC)(a0) csrr t0, CSR_STATUS REG_S t0, (SUSPEND_CONTEXT_REGS + PT_STATUS)(a0) + /* There is no need to save CSR_IE as it is maintained in memory */ csrr t0, CSR_TVAL REG_S t0, (SUSPEND_CONTEXT_REGS + PT_BADADDR)(a0) csrr t0, CSR_CAUSE diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c index 9a55e733ae99..bdc10be9d3b4 100644 --- a/drivers/clocksource/timer-clint.c +++ b/drivers/clocksource/timer-clint.c @@ -114,7 +114,9 @@ static int clint_clock_next_event(unsigned long delta, void __iomem *r = clint_timer_cmp + cpuid_to_hartid_map(smp_processor_id()); +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_set(CSR_IE, IE_TIE); +#endif writeq_relaxed(clint_get_cycles64() + delta, r); return 0; } @@ -155,7 +157,9 @@ static irqreturn_t clint_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evdev = this_cpu_ptr(&clint_clock_event); +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_clear(CSR_IE, IE_TIE); +#endif evdev->event_handler(evdev); return IRQ_HANDLED; diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index da3071b387eb..b730e01a7f02 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -36,7 +36,9 @@ static int riscv_clock_next_event(unsigned long delta, { u64 next_tval = get_cycles64() + delta; +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_set(CSR_IE, IE_TIE); +#endif if (static_branch_likely(&riscv_sstc_available)) { #if defined(CONFIG_32BIT) csr_write(CSR_STIMECMP, next_tval & 0xFFFFFFFF); @@ -119,7 +121,9 @@ static irqreturn_t riscv_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evdev = this_cpu_ptr(&riscv_clock_event); +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_clear(CSR_IE, IE_TIE); +#endif evdev->event_handler(evdev); return IRQ_HANDLED; diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c index e8d01b14ccdd..7fad1ba37e5c 100644 --- a/drivers/irqchip/irq-riscv-intc.c +++ b/drivers/irqchip/irq-riscv-intc.c @@ -39,12 +39,16 @@ static asmlinkage void riscv_intc_irq(struct pt_regs *regs) static void riscv_intc_irq_mask(struct irq_data *d) { +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_clear(CSR_IE, BIT(d->hwirq)); +#endif } static void riscv_intc_irq_unmask(struct irq_data *d) { +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_set(CSR_IE, BIT(d->hwirq)); +#endif } static void riscv_intc_irq_eoi(struct irq_data *d) diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 96c7f670c8f0..995b501ec721 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -778,7 +778,9 @@ static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) if (riscv_pmu_use_irq) { cpu_hw_evt->irq = riscv_pmu_irq; csr_clear(CSR_IP, BIT(riscv_pmu_irq_num)); +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_set(CSR_IE, BIT(riscv_pmu_irq_num)); +#endif enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); } @@ -789,7 +791,9 @@ static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) { if (riscv_pmu_use_irq) { disable_percpu_irq(riscv_pmu_irq); +#ifndef CONFIG_RISCV_PSEUDO_NMI csr_clear(CSR_IE, BIT(riscv_pmu_irq_num)); +#endif } /* Disable all counters access for user mode now */ -- 2.20.1
| |