Messages in this thread Patch in this message | | | From | Lai Jiangshan <> | Subject | [PATCH V5 33/50] x86/entry: Add the C version ist_restore_cr3() | Date | Wed, 10 Nov 2021 19:57:19 +0800 |
| |
From: Lai Jiangshan <laijs@linux.alibaba.com>
It implements the C version of RESTORE_CR3().
Not functional difference intended except the ASM code uses bit test and clear operations while the C version uses mask check and 'AND' operations. The resulted asm code of both versions are very similar.
Cc: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Lai Jiangshan <laijs@linux.alibaba.com> --- arch/x86/entry/entry64.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index 283bd685a275..5f47221d8935 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -11,6 +11,7 @@ * environments that the GS base is user controlled value, or the CR3 * is PTI user CR3 or both. */ +#include <asm/tlbflush.h> #include <asm/traps.h> extern unsigned char asm_load_gs_index_gs_change[]; @@ -30,6 +31,26 @@ static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr3) native_write_cr3(cr3); } +static __always_inline void pti_switch_to_user_cr3(unsigned long user_cr3) +{ +#define KERN_PCID_MASK (CR3_PCID_MASK & ~PTI_USER_PCID_MASK) + + if (static_cpu_has(X86_FEATURE_PCID)) { + int pcid = user_cr3 & KERN_PCID_MASK; + unsigned short pcid_mask = 1ull << pcid; + + /* + * Check if there's a pending flush for the user ASID we're + * about to set. + */ + if (!(this_cpu_read(cpu_tlbstate.user_pcid_flush_mask) & pcid_mask)) + user_cr3 |= X86_CR3_PCID_NOFLUSH; + else + this_cpu_and(cpu_tlbstate.user_pcid_flush_mask, ~pcid_mask); + } + native_write_cr3(user_cr3); +} + static __always_inline void switch_to_kernel_cr3(void) { if (static_cpu_has(X86_FEATURE_PTI)) @@ -49,9 +70,20 @@ static __always_inline unsigned long ist_switch_to_kernel_cr3(void) return cr3; } + +static __always_inline void ist_restore_cr3(unsigned long cr3) +{ + if (!static_cpu_has(X86_FEATURE_PTI)) + return; + + /* No need to restore when @cr3 is kernel CR3. */ + if (cr3 & PTI_USER_PGTABLE_MASK) + pti_switch_to_user_cr3(cr3); +} #else static __always_inline void switch_to_kernel_cr3(void) {} static __always_inline unsigned long ist_switch_to_kernel_cr3(void) { return 0; } +static __always_inline void ist_restore_cr3(unsigned long cr3) {} #endif /* -- 2.19.1.6.gb485710b
| |