Skip to content

Commit 0e2adab

Browse files
torvaldstorvalds
committed
Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 fixes from Will Deacon: "The main thing here is a long-awaited workaround for a CPU erratum on ThunderX2 which we have developed in conjunction with engineers from Cavium/Marvell. At the moment, the workaround is unconditionally enabled for affected CPUs at runtime but we may add a command-line option to disable it in future if performance numbers show up indicating a significant cost for real workloads. Summary: - Work around Cavium/Marvell ThunderX2 erratum hardkernel#219 - Fix regression in mlock() ABI caused by sign-extension of TTBR1 addresses - More fixes to the spurious kernel fault detection logic - Fix pathological preemption race when enabling some CPU features at boot - Drop broken kcore macros in favour of generic implementations - Fix userspace view of ID_AA64ZFR0_EL1 when SVE is disabled - Avoid NULL dereference on allocation failure during hibernation" * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: tags: Preserve tags for addresses translated via TTBR1 arm64: mm: fix inverted PAR_EL1.F check arm64: sysreg: fix incorrect definition of SYS_PAR_EL1_F arm64: entry.S: Do not preempt from IRQ before all cpufeatures are enabled arm64: hibernate: check pgd table allocation arm64: cpufeature: Treat ID_AA64ZFR0_EL1 as RAZ when SVE is not enabled arm64: Fix kcore macros after 52-bit virtual addressing fallout arm64: Allow CAVIUM_TX2_ERRATUM_219 to be selected arm64: Avoid Cavium TX2 erratum 219 when switching TTBR arm64: Enable workaround for Cavium TX2 erratum 219 when running SMT arm64: KVM: Trap VM ops when ARM64_WORKAROUND_CAVIUM_TX2_219_TVM is set
2 parents ad32fd7 + 777d062 commit 0e2adab

File tree

15 files changed

+186
-23
lines changed

15 files changed

+186
-23
lines changed

‎Documentation/arm64/silicon-errata.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ stable kernels.
107107
+----------------+-----------------+-----------------+-----------------------------+
108108
| Cavium | ThunderX2 SMMUv3| #126 | N/A |
109109
+----------------+-----------------+-----------------+-----------------------------+
110+
| Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 |
111+
+----------------+-----------------+-----------------+-----------------------------+
110112
+----------------+-----------------+-----------------+-----------------------------+
111113
| Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 |
112114
+----------------+-----------------+-----------------+-----------------------------+

‎arch/arm64/Kconfig

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -616,6 +616,23 @@ config CAVIUM_ERRATUM_30115
616616

617617
If unsure, say Y.
618618

619+
config CAVIUM_TX2_ERRATUM_219
620+
bool "Cavium ThunderX2 erratum 219: PRFM between TTBR change and ISB fails"
621+
default y
622+
help
623+
On Cavium ThunderX2, a load, store or prefetch instruction between a
624+
TTBR update and the corresponding context synchronizing operation can
625+
cause a spurious Data Abort to be delivered to any hardware thread in
626+
the CPU core.
627+
628+
Work around the issue by avoiding the problematic code sequence and
629+
trapping KVM guest TTBRx_EL1 writes to EL2 when SMT is enabled. The
630+
trap handler performs the corresponding register access, skips the
631+
instruction and ensures context synchronization by virtue of the
632+
exception return.
633+
634+
If unsure, say Y.
635+
619636
config QCOM_FALKOR_ERRATUM_1003
620637
bool "Falkor E1003: Incorrect translation due to ASID change"
621638
default y

‎arch/arm64/include/asm/asm-uaccess.h

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,9 @@ alternative_else_nop_endif
7878
/*
7979
* Remove the address tag from a virtual address, if present.
8080
*/
81-
.macro clear_address_tag, dst, addr
82-
tst \addr, #(1 << 55)
83-
bic \dst, \addr, #(0xff << 56)
84-
csel \dst, \dst, \addr, eq
81+
.macro untagged_addr, dst, addr
82+
sbfx \dst, \addr, #0, #56
83+
and \dst, \dst, \addr
8584
.endm
8685

8786
#endif

‎arch/arm64/include/asm/cpucaps.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@
5252
#define ARM64_HAS_IRQ_PRIO_MASKING 42
5353
#define ARM64_HAS_DCPODP 43
5454
#define ARM64_WORKAROUND_1463225 44
55+
#define ARM64_WORKAROUND_CAVIUM_TX2_219_TVM 45
56+
#define ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM 46
5557

56-
#define ARM64_NCAPS 45
58+
#define ARM64_NCAPS 47
5759

5860
#endif /* __ASM_CPUCAPS_H */

‎arch/arm64/include/asm/memory.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,18 @@ static inline unsigned long kaslr_offset(void)
215215
* up with a tagged userland pointer. Clear the tag to get a sane pointer to
216216
* pass on to access_ok(), for instance.
217217
*/
218-
#define untagged_addr(addr) \
218+
#define __untagged_addr(addr) \
219219
((__force __typeof__(addr))sign_extend64((__force u64)(addr), 55))
220220

221+
#define untagged_addr(addr) ({ \
222+
u64 __addr = (__force u64)addr; \
223+
__addr &= __untagged_addr(__addr); \
224+
(__force __typeof__(addr))__addr; \
225+
})
226+
221227
#ifdef CONFIG_KASAN_SW_TAGS
222228
#define __tag_shifted(tag) ((u64)(tag) << 56)
223-
#define __tag_reset(addr) untagged_addr(addr)
229+
#define __tag_reset(addr) __untagged_addr(addr)
224230
#define __tag_get(addr) (__u8)((u64)(addr) >> 56)
225231
#else
226232
#define __tag_shifted(tag) 0UL

‎arch/arm64/include/asm/pgtable.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -876,9 +876,6 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
876876

877877
#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
878878

879-
#define kc_vaddr_to_offset(v) ((v) & ~PAGE_END)
880-
#define kc_offset_to_vaddr(o) ((o) | PAGE_END)
881-
882879
#ifdef CONFIG_ARM64_PA_BITS_52
883880
#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
884881
#else

‎arch/arm64/include/asm/sysreg.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@
212212
#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
213213
#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
214214

215-
#define SYS_PAR_EL1_F BIT(1)
215+
#define SYS_PAR_EL1_F BIT(0)
216216
#define SYS_PAR_EL1_FST GENMASK(6, 1)
217217

218218
/*** Statistical Profiling Extension ***/

‎arch/arm64/kernel/cpu_errata.c

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <asm/cpu.h>
1313
#include <asm/cputype.h>
1414
#include <asm/cpufeature.h>
15+
#include <asm/smp_plat.h>
1516

1617
static bool __maybe_unused
1718
is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
@@ -623,6 +624,30 @@ check_branch_predictor(const struct arm64_cpu_capabilities *entry, int scope)
623624
return (need_wa > 0);
624625
}
625626

627+
static const __maybe_unused struct midr_range tx2_family_cpus[] = {
628+
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
629+
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
630+
{},
631+
};
632+
633+
static bool __maybe_unused
634+
needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry,
635+
int scope)
636+
{
637+
int i;
638+
639+
if (!is_affected_midr_range_list(entry, scope) ||
640+
!is_hyp_mode_available())
641+
return false;
642+
643+
for_each_possible_cpu(i) {
644+
if (MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0) != 0)
645+
return true;
646+
}
647+
648+
return false;
649+
}
650+
626651
#ifdef CONFIG_HARDEN_EL2_VECTORS
627652

628653
static const struct midr_range arm64_harden_el2_vectors[] = {
@@ -851,6 +876,19 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
851876
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
852877
.matches = has_cortex_a76_erratum_1463225,
853878
},
879+
#endif
880+
#ifdef CONFIG_CAVIUM_TX2_ERRATUM_219
881+
{
882+
.desc = "Cavium ThunderX2 erratum 219 (KVM guest sysreg trapping)",
883+
.capability = ARM64_WORKAROUND_CAVIUM_TX2_219_TVM,
884+
ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
885+
.matches = needs_tx2_tvm_workaround,
886+
},
887+
{
888+
.desc = "Cavium ThunderX2 erratum 219 (PRFM removal)",
889+
.capability = ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM,
890+
ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
891+
},
854892
#endif
855893
{
856894
}

‎arch/arm64/kernel/cpufeature.c

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,11 +176,16 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
176176
};
177177

178178
static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
179-
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SM4_SHIFT, 4, 0),
180-
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SHA3_SHIFT, 4, 0),
181-
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_BITPERM_SHIFT, 4, 0),
182-
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_AES_SHIFT, 4, 0),
183-
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SVEVER_SHIFT, 4, 0),
179+
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
180+
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SM4_SHIFT, 4, 0),
181+
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
182+
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SHA3_SHIFT, 4, 0),
183+
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
184+
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_BITPERM_SHIFT, 4, 0),
185+
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
186+
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_AES_SHIFT, 4, 0),
187+
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
188+
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SVEVER_SHIFT, 4, 0),
184189
ARM64_FTR_END,
185190
};
186191

‎arch/arm64/kernel/entry.S

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,7 @@ el1_da:
604604
*/
605605
mrs x3, far_el1
606606
inherit_daif pstate=x23, tmp=x2
607-
clear_address_tag x0, x3
607+
untagged_addr x0, x3
608608
mov x2, sp // struct pt_regs
609609
bl do_mem_abort
610610

@@ -680,7 +680,7 @@ alternative_if ARM64_HAS_IRQ_PRIO_MASKING
680680
orr x24, x24, x0
681681
alternative_else_nop_endif
682682
cbnz x24, 1f // preempt count != 0 || NMI return path
683-
bl preempt_schedule_irq // irq en/disable is done inside
683+
bl arm64_preempt_schedule_irq // irq en/disable is done inside
684684
1:
685685
#endif
686686

@@ -808,7 +808,7 @@ el0_da:
808808
mrs x26, far_el1
809809
ct_user_exit_irqoff
810810
enable_daif
811-
clear_address_tag x0, x26
811+
untagged_addr x0, x26
812812
mov x1, x25
813813
mov x2, sp
814814
bl do_mem_abort
@@ -1071,7 +1071,9 @@ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
10711071
#else
10721072
ldr x30, =vectors
10731073
#endif
1074+
alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
10741075
prfm plil1strm, [x30, #(1b - tramp_vectors)]
1076+
alternative_else_nop_endif
10751077
msr vbar_el1, x30
10761078
add x30, x30, #(1b - tramp_vectors)
10771079
isb

‎arch/arm64/kernel/hibernate.c

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ static int create_safe_exec_page(void *src_start, size_t length,
201201
gfp_t mask)
202202
{
203203
int rc = 0;
204+
pgd_t *trans_pgd;
204205
pgd_t *pgdp;
205206
pud_t *pudp;
206207
pmd_t *pmdp;
@@ -215,7 +216,13 @@ static int create_safe_exec_page(void *src_start, size_t length,
215216
memcpy((void *)dst, src_start, length);
216217
__flush_icache_range(dst, dst + length);
217218

218-
pgdp = pgd_offset_raw(allocator(mask), dst_addr);
219+
trans_pgd = allocator(mask);
220+
if (!trans_pgd) {
221+
rc = -ENOMEM;
222+
goto out;
223+
}
224+
225+
pgdp = pgd_offset_raw(trans_pgd, dst_addr);
219226
if (pgd_none(READ_ONCE(*pgdp))) {
220227
pudp = allocator(mask);
221228
if (!pudp) {

‎arch/arm64/kernel/process.c

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <linux/sched/task.h>
1818
#include <linux/sched/task_stack.h>
1919
#include <linux/kernel.h>
20+
#include <linux/lockdep.h>
2021
#include <linux/mm.h>
2122
#include <linux/stddef.h>
2223
#include <linux/sysctl.h>
@@ -44,6 +45,7 @@
4445
#include <asm/alternative.h>
4546
#include <asm/arch_gicv3.h>
4647
#include <asm/compat.h>
48+
#include <asm/cpufeature.h>
4749
#include <asm/cacheflush.h>
4850
#include <asm/exec.h>
4951
#include <asm/fpsimd.h>
@@ -631,3 +633,19 @@ static int __init tagged_addr_init(void)
631633

632634
core_initcall(tagged_addr_init);
633635
#endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */
636+
637+
asmlinkage void __sched arm64_preempt_schedule_irq(void)
638+
{
639+
lockdep_assert_irqs_disabled();
640+
641+
/*
642+
* Preempting a task from an IRQ means we leave copies of PSTATE
643+
* on the stack. cpufeature's enable calls may modify PSTATE, but
644+
* resuming one of these preempted tasks would undo those changes.
645+
*
646+
* Only allow a task to be preempted once cpufeatures have been
647+
* enabled.
648+
*/
649+
if (static_branch_likely(&arm64_const_caps_ready))
650+
preempt_schedule_irq();
651+
}

‎arch/arm64/kvm/hyp/switch.c

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
124124
{
125125
u64 hcr = vcpu->arch.hcr_el2;
126126

127+
if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
128+
hcr |= HCR_TVM;
129+
127130
write_sysreg(hcr, hcr_el2);
128131

129132
if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
@@ -174,8 +177,10 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
174177
* the crucial bit is "On taking a vSError interrupt,
175178
* HCR_EL2.VSE is cleared to 0."
176179
*/
177-
if (vcpu->arch.hcr_el2 & HCR_VSE)
178-
vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);
180+
if (vcpu->arch.hcr_el2 & HCR_VSE) {
181+
vcpu->arch.hcr_el2 &= ~HCR_VSE;
182+
vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
183+
}
179184

180185
if (has_vhe())
181186
deactivate_traps_vhe();
@@ -380,6 +385,61 @@ static bool __hyp_text __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)
380385
return true;
381386
}
382387

388+
static bool __hyp_text handle_tx2_tvm(struct kvm_vcpu *vcpu)
389+
{
390+
u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_hsr(vcpu));
391+
int rt = kvm_vcpu_sys_get_rt(vcpu);
392+
u64 val = vcpu_get_reg(vcpu, rt);
393+
394+
/*
395+
* The normal sysreg handling code expects to see the traps,
396+
* let's not do anything here.
397+
*/
398+
if (vcpu->arch.hcr_el2 & HCR_TVM)
399+
return false;
400+
401+
switch (sysreg) {
402+
case SYS_SCTLR_EL1:
403+
write_sysreg_el1(val, SYS_SCTLR);
404+
break;
405+
case SYS_TTBR0_EL1:
406+
write_sysreg_el1(val, SYS_TTBR0);
407+
break;
408+
case SYS_TTBR1_EL1:
409+
write_sysreg_el1(val, SYS_TTBR1);
410+
break;
411+
case SYS_TCR_EL1:
412+
write_sysreg_el1(val, SYS_TCR);
413+
break;
414+
case SYS_ESR_EL1:
415+
write_sysreg_el1(val, SYS_ESR);
416+
break;
417+
case SYS_FAR_EL1:
418+
write_sysreg_el1(val, SYS_FAR);
419+
break;
420+
case SYS_AFSR0_EL1:
421+
write_sysreg_el1(val, SYS_AFSR0);
422+
break;
423+
case SYS_AFSR1_EL1:
424+
write_sysreg_el1(val, SYS_AFSR1);
425+
break;
426+
case SYS_MAIR_EL1:
427+
write_sysreg_el1(val, SYS_MAIR);
428+
break;
429+
case SYS_AMAIR_EL1:
430+
write_sysreg_el1(val, SYS_AMAIR);
431+
break;
432+
case SYS_CONTEXTIDR_EL1:
433+
write_sysreg_el1(val, SYS_CONTEXTIDR);
434+
break;
435+
default:
436+
return false;
437+
}
438+
439+
__kvm_skip_instr(vcpu);
440+
return true;
441+
}
442+
383443
/*
384444
* Return true when we were able to fixup the guest exit and should return to
385445
* the guest, false when we should restore the host state and return to the
@@ -399,6 +459,11 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
399459
if (*exit_code != ARM_EXCEPTION_TRAP)
400460
goto exit;
401461

462+
if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
463+
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 &&
464+
handle_tx2_tvm(vcpu))
465+
return true;
466+
402467
/*
403468
* We trap the first access to the FP/SIMD to save the host context
404469
* and restore the guest context lazily.

‎arch/arm64/mm/fault.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,12 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
268268
par = read_sysreg(par_el1);
269269
local_irq_restore(flags);
270270

271+
/*
272+
* If we now have a valid translation, treat the translation fault as
273+
* spurious.
274+
*/
271275
if (!(par & SYS_PAR_EL1_F))
272-
return false;
276+
return true;
273277

274278
/*
275279
* If we got a different type of fault from the AT instruction,

‎include/linux/sched.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ extern long schedule_timeout_uninterruptible(long timeout);
223223
extern long schedule_timeout_idle(long timeout);
224224
asmlinkage void schedule(void);
225225
extern void schedule_preempt_disabled(void);
226+
asmlinkage void preempt_schedule_irq(void);
226227

227228
extern int __must_check io_schedule_prepare(void);
228229
extern void io_schedule_finish(int token);

0 commit comments

Comments
 (0)