From a36b6ec0f960b50337107e0d7555dc430e6aa08b Mon Sep 17 00:00:00 2001 From: Michal Meloun Date: Sat, 27 Jan 2018 11:19:41 +0000 Subject: [PATCH] Implement mitigation for Spectre version 2 attacks on ARMv7. Similarly as we already do for arm64, for mitigation is necessary to flush branch predictor when we: - do task switch - receive prefetch abort on non-userspace address The user can disable this mitigation by setting 'machdep.disable_bp_hardening' sysctl variable, or it can check actual system status by reading 'machdep.spectre_v2_safe' The situation is complicated by fact that: - for Cortex-A8, the BPIALL instruction is effectively NOP until the IBE bit in ACTLR is set. - for Cortex-A15, the BPIALL is always NOP. The branch predictor can be only flushed by doing ICIALLU with special bit (Enable invalidates of BTB) set in ACTLR. Since access to the ACTLR register is locked to secure monitor/firmware on most boards, they will also need update of firmware / U-boot. In worst case, when secure monitor is on-chip ROM (e.g. PandaBoard), the board is unfixable. MFC after: 2 weeks Reviewed by: imp, emaste Differential Revision: https://reviews.freebsd.org/D13931 --- sys/arm/arm/cpuinfo.c | 199 ++++++++++++++++++++++++++++++++++++++ sys/arm/arm/genassym.c | 5 + sys/arm/arm/machdep.c | 2 + sys/arm/arm/mp_machdep.c | 3 + sys/arm/arm/swtch-v6.S | 11 ++- sys/arm/arm/trap-v6.c | 15 +++ sys/arm/include/cpuinfo.h | 2 + sys/arm/include/pcpu.h | 8 +- 8 files changed, 243 insertions(+), 2 deletions(-) diff --git a/sys/arm/arm/cpuinfo.c b/sys/arm/arm/cpuinfo.c index 15efa9fd378..4b7c21e8da3 100644 --- a/sys/arm/arm/cpuinfo.c +++ b/sys/arm/arm/cpuinfo.c @@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include +#include #include #include @@ -40,6 +42,9 @@ __FBSDID("$FreeBSD$"); #if __ARM_ARCH >= 6 void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set); + +int disable_bp_hardening; +int spectre_v2_safe = 1; #endif struct cpuinfo cpuinfo = @@ -255,6 +260,7 @@ cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint32_t *actlr_set) if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { switch (cpuinfo.part_number) { + case CPU_ARCH_CORTEX_A75: case CPU_ARCH_CORTEX_A73: case CPU_ARCH_CORTEX_A72: case CPU_ARCH_CORTEX_A57: @@ -338,4 +344,197 @@ cpuinfo_reinit_mmu(uint32_t ttb) reinit_mmu(ttb, actlr_mask, actlr_set); } +static bool +modify_actlr(uint32_t clear, uint32_t set) +{ + uint32_t reg, newreg; + + reg = cp15_actlr_get(); + newreg = reg; + newreg &= ~clear; + newreg |= set; + if (reg == newreg) + return (true); + cp15_actlr_set(newreg); + + reg = cp15_actlr_get(); + if (reg == newreg) + return (true); + return (false); +} + +/* Apply/restore BP hardening on current core. */ +static int +apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask) +{ + if (enable) { + if (actrl && !modify_actlr(0, set_mask)) + return (-1); + PCPU_SET(bp_harden_kind, kind); + } else { + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + if (actrl) + modify_actlr(~0, PCPU_GET(original_actlr)); + spectre_v2_safe = 0; + } + return (0); +} + +static void +handle_bp_hardening(bool enable) +{ + int kind; + char *kind_str; + + kind = PCPU_BP_HARDEN_KIND_NONE; + /* + * Note: Access to ACTRL is locked to secure world on most boards. + * This means that full BP hardening depends on updated u-boot/firmware + * or is impossible at all (if secure monitor is in on-chip ROM). + */ + if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { + switch (cpuinfo.part_number) { + case CPU_ARCH_CORTEX_A8: + /* + * For Cortex-A8, IBE bit must be set otherwise + * BPIALL is effectively NOP. + * Unfortunately, Cortex-A is also affected by + * ARM erratum 687067 which causes non-working + * BPIALL if IBE bit is set and 'Instruction L1 System + * Array Debug Register 0' is not 0. + * This register is not reset on power-up and is + * accessible only from secure world, so we cannot do + * nothing (nor detect) to fix this issue. + * I afraid that on chip ROM based secure monitor on + * AM335x (BeagleBone) doesn't reset this debug + * register. + */ + kind = PCPU_BP_HARDEN_KIND_BPIALL; + if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0) + goto actlr_err; + break; + break; + + case CPU_ARCH_CORTEX_A9: + case CPU_ARCH_CORTEX_A12: + case CPU_ARCH_CORTEX_A17: + case CPU_ARCH_CORTEX_A57: + case CPU_ARCH_CORTEX_A72: + case CPU_ARCH_CORTEX_A73: + case CPU_ARCH_CORTEX_A75: + kind = PCPU_BP_HARDEN_KIND_BPIALL; + if (apply_bp_hardening(enable, kind, false, 0) != 0) + goto actlr_err; + break; + + case CPU_ARCH_CORTEX_A15: + /* + * For Cortex-A15, set 'Enable invalidates of BTB' bit. + * Despite this, the BPIALL is still effectively NOP, + * but with this bit set, the ICIALLU also flushes + * branch predictor as side effect. + */ + kind = PCPU_BP_HARDEN_KIND_ICIALLU; + if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0) + goto actlr_err; + break; + + default: + break; + } + } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) { + printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative " + "branch attacks. !!!\n" + "Qualcomm Krait cores are known (or believed) to be " + "vulnerable to \n" + "speculative branch attacks, no mitigation exists yet.\n", + PCPU_GET(cpuid)); + goto unkonown_mitigation; + } else { + goto unkonown_mitigation; + } + + if (bootverbose) { + switch (kind) { + case PCPU_BP_HARDEN_KIND_NONE: + kind_str = "not necessary"; + break; + case PCPU_BP_HARDEN_KIND_BPIALL: + kind_str = "BPIALL"; + break; + case PCPU_BP_HARDEN_KIND_ICIALLU: + kind_str = "ICIALLU"; + break; + default: + panic("Unknown BP hardering kind (%d).", kind); + } + printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid), + kind_str); + } + + return; + +unkonown_mitigation: + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + spectre_v2_safe = 0; + return; + +actlr_err: + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + spectre_v2_safe = 0; + printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch " + "attacks. !!!\n" + "We cannot enable required bit(s) in ACTRL register\n" + "because it's locked by secure monitor and/or firmware.\n", + PCPU_GET(cpuid)); +} + +void +cpuinfo_init_bp_hardening(void) +{ + + /* + * Store original unmodified ACTRL, so we can restore it when + * BP hardening is disabled by sysctl. + */ + PCPU_SET(original_actlr, cp15_actlr_get()); + handle_bp_hardening(true); +} + +static void +bp_hardening_action(void *arg) +{ + + handle_bp_hardening(disable_bp_hardening == 0); +} + +static int +sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS) +{ + int rv; + + rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); + + if (!rv && req->newptr) { + spectre_v2_safe = 1; + dmb(); +#ifdef SMP + smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, + bp_hardening_action, NULL, NULL); +#else + bp_hardening_action(NULL); +#endif + } + + return (rv); +} + +SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I", + "Disable BP hardening mitigation."); + +SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD, + &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks"); + #endif /* __ARM_ARCH >= 6 */ diff --git a/sys/arm/arm/genassym.c b/sys/arm/arm/genassym.c index 6884b803701..f987a582255 100644 --- a/sys/arm/arm/genassym.c +++ b/sys/arm/arm/genassym.c @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -136,6 +137,10 @@ ASSYM(PCB_VFPSTATE, offsetof(struct pcb, pcb_vfpstate)); #if __ARM_ARCH >= 6 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap)); +ASSYM(PC_BP_HARDEN_KIND, offsetof(struct pcpu, pc_bp_harden_kind)); +ASSYM(PCPU_BP_HARDEN_KIND_NONE, PCPU_BP_HARDEN_KIND_NONE); +ASSYM(PCPU_BP_HARDEN_KIND_BPIALL, PCPU_BP_HARDEN_KIND_BPIALL); +ASSYM(PCPU_BP_HARDEN_KIND_ICIALLU, PCPU_BP_HARDEN_KIND_ICIALLU); #endif ASSYM(PAGE_SIZE, PAGE_SIZE); diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c index 474fa82df58..4fdb37ef518 100644 --- a/sys/arm/arm/machdep.c +++ b/sys/arm/arm/machdep.c @@ -1264,6 +1264,8 @@ initarm(struct arm_boot_params *abp) msgbufinit(msgbufp, msgbufsize); dbg_monitor_init(); kdb_init(); + /* Apply possible BP hardening. */ + cpuinfo_init_bp_hardening(); return ((void *)STACKALIGN(thread0.td_pcb)); } diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c index 4bd2b919318..b0eb678394c 100644 --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -201,6 +201,9 @@ init_secondary(int cpu) /* Configure the interrupt controller */ intr_pic_init_secondary(); + /* Apply possible BP hardening */ + cpuinfo_init_bp_hardening(); + mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); diff --git a/sys/arm/arm/swtch-v6.S b/sys/arm/arm/swtch-v6.S index 8bbaac9df74..9a97a789bca 100644 --- a/sys/arm/arm/swtch-v6.S +++ b/sys/arm/arm/swtch-v6.S @@ -145,7 +145,16 @@ ENTRY(cpu_context_switch) * predictors and Requirements for branch predictor maintenance * operations sections. */ - mcr CP15_BPIALL /* flush entire Branch Target Cache */ + /* + * Additionally, to mitigate mistrained branch predictor attack + * we must invalidate it on affected CPUs. Unfortunately, BPIALL + * is effectively NOP on Cortex-A15 so it needs special treatment. + */ + ldr r0, [r8, #PC_BP_HARDEN_KIND] + cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU + mcrne CP15_BPIALL /* Flush entire Branch Target Cache */ + mcreq CP15_ICIALLU /* This is the only way how to flush */ + /* Branch Target Cache on Cortex-A15. */ DSB mov pc, lr END(cpu_context_switch) diff --git a/sys/arm/arm/trap-v6.c b/sys/arm/arm/trap-v6.c index a6db6dca141..d3efc0c0ee5 100644 --- a/sys/arm/arm/trap-v6.c +++ b/sys/arm/arm/trap-v6.c @@ -287,6 +287,7 @@ abort_handler(struct trapframe *tf, int prefetch) struct vmspace *vm; vm_prot_t ftype; bool usermode; + int bp_harden; #ifdef INVARIANTS void *onfault; #endif @@ -303,6 +304,20 @@ abort_handler(struct trapframe *tf, int prefetch) idx = FSR_TO_FAULT(fsr); usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */ + + /* + * Apply BP hardening by flushing the branch prediction cache + * for prefaults on kernel addresses. + */ + if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS && + (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) { + bp_harden = PCPU_GET(bp_harden_kind); + if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL) + _CP15_BPIALL(); + else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU) + _CP15_ICIALLU(); + } + if (usermode) td->td_frame = tf; diff --git a/sys/arm/include/cpuinfo.h b/sys/arm/include/cpuinfo.h index 943391ada4d..35c496db43f 100644 --- a/sys/arm/include/cpuinfo.h +++ b/sys/arm/include/cpuinfo.h @@ -49,6 +49,7 @@ #define CPU_ARCH_CORTEX_A57 0xD07 #define CPU_ARCH_CORTEX_A72 0xD08 #define CPU_ARCH_CORTEX_A73 0xD09 +#define CPU_ARCH_CORTEX_A75 0xD0A /* QCOM */ @@ -125,6 +126,7 @@ extern struct cpuinfo cpuinfo; void cpuinfo_init(void); #if __ARM_ARCH >= 6 +void cpuinfo_init_bp_hardening(void); void cpuinfo_reinit_mmu(uint32_t ttb); #endif #endif /* _MACHINE_CPUINFO_H_ */ diff --git a/sys/arm/include/pcpu.h b/sys/arm/include/pcpu.h index d72741a84d9..ea2ba540339 100644 --- a/sys/arm/include/pcpu.h +++ b/sys/arm/include/pcpu.h @@ -44,6 +44,10 @@ struct vmspace; #endif /* _KERNEL */ #if __ARM_ARCH >= 6 +/* Branch predictor hardening method */ +#define PCPU_BP_HARDEN_KIND_NONE 0 +#define PCPU_BP_HARDEN_KIND_BPIALL 1 +#define PCPU_BP_HARDEN_KIND_ICIALLU 2 #define PCPU_MD_FIELDS \ unsigned int pc_vfpsid; \ @@ -59,7 +63,9 @@ struct vmspace; void *pc_qmap_pte2p; \ unsigned int pc_dbreg[32]; \ int pc_dbreg_cmd; \ - char __pad[155] + int pc_bp_harden_kind; \ + uint32_t pc_original_actlr; \ + char __pad[147] #else #define PCPU_MD_FIELDS \ char __pad[93]