--- sys/arm64/arm64/pmap.c.orig +++ sys/arm64/arm64/pmap.c @@ -183,6 +183,8 @@ #define PMAP_SAN_PTE_BITS (ATTR_DEFAULT | ATTR_S1_XN | \ ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) +static bool __read_mostly pmap_multiple_tlbi = false; + struct pmap_large_md_page { struct rwlock pv_lock; struct md_page pv_page; @@ -1534,6 +1536,71 @@ } } + +static void +pmap_init_multiple_tlbi(void *dummy __unused) +{ + u_int cpu, midr; + + CPU_FOREACH(cpu) { + midr = pcpu_find(cpu)->pc_midr; + + /* + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + * Present in all revisions + */ + if (CPU_IMPL(midr) == CPU_IMPL_ARM) { + switch(CPU_PART(midr)) { + case CPU_PART_C1_PREMIUM: + case CPU_PART_C1_ULTRA: + case CPU_PART_CORTEX_A76: + case CPU_PART_CORTEX_A76AE: + case CPU_PART_CORTEX_A77: + case CPU_PART_CORTEX_A78: + case CPU_PART_CORTEX_A78AE: + case CPU_PART_CORTEX_A78C: + case CPU_PART_CORTEX_A710: + case CPU_PART_CORTEX_X1: + case CPU_PART_CORTEX_X1C: + case CPU_PART_CORTEX_X2: + case CPU_PART_CORTEX_X3: + case CPU_PART_CORTEX_X4: + case CPU_PART_CORTEX_X925: + case CPU_PART_NEOVERSE_N1: + case CPU_PART_NEOVERSE_N2: + case CPU_PART_NEOVERSE_V1: + case CPU_PART_NEOVERSE_V2: + case CPU_PART_NEOVERSE_V3: + case CPU_PART_NEOVERSE_V3AE: + pmap_multiple_tlbi = true; + return; + } + } + } +} +SYSINIT(pmap_init_multiple_tlbi, SI_SUB_CPU, SI_ORDER_ANY, + pmap_init_multiple_tlbi, NULL); + /* * Initialize the pmap module. * @@ -1652,6 +1719,10 @@ r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); pmap_s1_invalidate_user(r, final_only); } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); + } dsb(ish); isb(); } @@ -1699,6 +1770,10 @@ for (r = start; r < end; r += TLBI_VA_L3_INCR) pmap_s1_invalidate_user(r, final_only); } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); + } dsb(ish); isb(); } @@ -1740,6 +1815,10 @@ r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); __asm __volatile("tlbi aside1is, %0" : : "r" (r)); } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); + } dsb(ish); isb(); } --- sys/arm64/include/cpu.h.orig +++ sys/arm64/include/cpu.h @@ -77,6 +77,7 @@ #define CPU_IMPL_CAVIUM 0x43 #define CPU_IMPL_DEC 0x44 #define CPU_IMPL_FUJITSU 0x46 +#define CPU_IMPL_HISILICON 0x48 #define CPU_IMPL_INFINEON 0x49 #define CPU_IMPL_FREESCALE 0x4D #define CPU_IMPL_NVIDIA 0x4E @@ -86,6 +87,7 @@ #define CPU_IMPL_APPLE 0x61 #define CPU_IMPL_INTEL 0x69 #define CPU_IMPL_AMPERE 0xC0 +#define CPU_IMPL_MICROSOFT 0x6D /* ARM Part numbers */ #define CPU_PART_FOUNDATION 0xD00 @@ -105,6 +107,7 @@ #define CPU_PART_AEM_V8 0xD0F #define CPU_PART_NEOVERSE_V1 0xD40 #define CPU_PART_CORTEX_A78 0xD41 +#define CPU_PART_CORTEX_A78AE 0xD42 #define CPU_PART_CORTEX_A65AE 0xD43 #define CPU_PART_CORTEX_X1 0xD44 #define CPU_PART_CORTEX_A510 0xD46 @@ -117,6 +120,18 @@ #define CPU_PART_CORTEX_A715 0xD4D #define CPU_PART_CORTEX_X3 0xD4E #define CPU_PART_NEOVERSE_V2 0xD4F +#define CPU_PART_CORTEX_A520 0xD80 +#define CPU_PART_CORTEX_A720 0xD81 +#define CPU_PART_CORTEX_X4 0xD82 +#define CPU_PART_NEOVERSE_V3AE 0xD83 +#define CPU_PART_NEOVERSE_V3 0xD84 +#define CPU_PART_CORTEX_X925 0xD85 +#define CPU_PART_CORTEX_A725 0xD87 +#define CPU_PART_C1_NANO 0xD8A +#define CPU_PART_C1_PRO 0xD8B +#define CPU_PART_C1_ULTRA 0xD8C +#define CPU_PART_NEOVERSE_N3 0xD8E +#define CPU_PART_C1_PREMIUM 0xD90 /* Cavium Part numbers */ #define CPU_PART_THUNDERX 0x0A1 @@ -129,9 +144,16 @@ #define CPU_REV_THUNDERX2_0 0x00 -/* APM / Ampere Part Number */ +/* APM (now Ampere) Part number */ #define CPU_PART_EMAG8180 0x000 +/* Ampere Part numbers */ +#define CPU_PART_AMPERE1 0xAC3 +#define CPU_PART_AMPERE1A 0xAC4 + +/* Microsoft Part numbers */ +#define CPU_PART_AZURE_COBALT_100 0xD49 + /* Qualcomm */ #define CPU_PART_KRYO400_GOLD 0x804 #define CPU_PART_KRYO400_SILVER 0x805