diff options
Diffstat (limited to 'sys/arch/amd64/amd64/machdep.c')
| -rw-r--r-- | sys/arch/amd64/amd64/machdep.c | 226 |
1 files changed, 212 insertions, 14 deletions
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c index a5fb4bf..40950f9 100644 --- a/sys/arch/amd64/amd64/machdep.c +++ b/sys/arch/amd64/amd64/machdep.c @@ -40,7 +40,22 @@ #include <machine/cpuid.h> #include <machine/lapic.h> #include <machine/uart.h> +#include <machine/sync.h> #include <machine/intr.h> +#include <machine/cdefs.h> +#include <machine/isa/i8042var.h> +#include <dev/cons/cons.h> +#include <string.h> + +#define pr_trace(fmt, ...) kprintf("cpu: " fmt, ##__VA_ARGS__) +#define pr_error(...) pr_trace(__VA_ARGS__) +#define pr_trace_bsp(...) \ + if (!bsp_init) { \ + pr_trace(__VA_ARGS__); \ + } + +#define HALT_VECTOR 0x21 +#define TLB_VECTOR 0x22 #if defined(__SPECTRE_IBRS) #define SPECTRE_IBRS __SPECTRE_IBRS @@ -48,16 +63,19 @@ #define SPECTRE_IBRS 0 #endif -static uint8_t halt_vector = 0; +#if defined(__CPU_SMEP) +#define CPU_SMEP __CPU_SMEP +#else +#define CPU_SMEP 0 +#endif int ibrs_enable(void); +int simd_init(void); void syscall_isr(void); +void pin_isr_load(void); struct cpu_info g_bsp_ci = {0}; -static struct gdtr bsp_gdtr = { - .limit = sizeof(struct gdt_entry) * 256 - 1, - .offset = (uintptr_t)&g_gdt_data[0] -}; +static bool bsp_init = false; __attribute__((__interrupt__)) static void @@ -67,13 +85,34 @@ cpu_halt_isr(void *p) __builtin_unreachable(); } +__attribute__((__interrupt__)) static void -setup_vectors(void) +tlb_shootdown_isr(void *p) { - if (halt_vector == 0) { - halt_vector = intr_alloc_vector("cpu-halt", IPL_HIGH); + struct cpu_info *ci; + int ipl; + + /* + * Get the current CPU and check if we even + * need a shootdown. If `tlb_shootdown' is + * unset, this is not for us. + */ + ci = this_cpu(); + if (!ci->tlb_shootdown) { + return; } + ipl = splraise(IPL_HIGH); + __invlpg(ci->shootdown_va); + + ci->shootdown_va = 0; + ci->tlb_shootdown = 0; + splx(ipl); +} + +static void +setup_vectors(void) +{ idt_set_desc(0x0, IDT_TRAP_GATE, ISR(arith_err), 0); idt_set_desc(0x2, IDT_TRAP_GATE, ISR(nmi), 0); idt_set_desc(0x3, IDT_TRAP_GATE, ISR(breakpoint_handler), 0); @@ -87,7 +126,9 @@ setup_vectors(void) idt_set_desc(0xD, IDT_TRAP_GATE, ISR(general_prot), 0); idt_set_desc(0xE, IDT_TRAP_GATE, ISR(page_fault), 0); idt_set_desc(0x80, IDT_USER_INT_GATE, ISR(syscall_isr), 0); - idt_set_desc(halt_vector, IDT_INT_GATE, ISR(cpu_halt_isr), 0); + idt_set_desc(HALT_VECTOR, IDT_INT_GATE, ISR(cpu_halt_isr), 0); + idt_set_desc(TLB_VECTOR, IDT_INT_GATE, ISR(tlb_shootdown_isr), 0); + pin_isr_load(); } static inline void @@ -95,7 +136,7 @@ init_tss(struct cpu_info *ci) { struct tss_desc *desc; - desc = (struct tss_desc *)&g_gdt_data[GDT_TSS]; + desc = (struct tss_desc *)&g_gdt_data[GDT_TSS_INDEX]; write_tss(ci, desc); tss_load(); } @@ -131,6 +172,81 @@ backtrace_addr_to_name(uintptr_t addr, off_t *off) return NULL; } +static void +enable_simd(void) +{ + int retval; + + if ((retval = simd_init()) < 0) { + pr_trace_bsp("SIMD not supported\n"); + } + + if (retval == 1) { + pr_trace_bsp("SSE enabled but not AVX\n"); + } +} + +static void +cpu_get_info(struct cpu_info *ci) +{ + uint32_t eax, ebx, unused; + uint8_t ext_model, ext_family; + + /* Extended features */ + CPUID(0x07, unused, ebx, unused, unused); + if (ISSET(ebx, BIT(7))) + ci->feat |= CPU_FEAT_SMEP; + if (ISSET(ebx, BIT(20))) + ci->feat |= CPU_FEAT_SMAP; + + /* + * Processor info and feature bits + */ + CPUID(0x01, eax, unused, unused, unused); + ci->model = (eax >> 4) & 0xF; + ci->family = (eax >> 8) & 0xF; + + /* + * If the family ID is 15 then the actual family + * ID is the sum of the extended family and the + * family ID fields. + */ + if (ci->family == 0xF) { + ext_family = (eax >> 20) & 0xFF; + ci->family += ext_family; + } + + /* + * If the family has the value of either 6 or 15, + * then the extended model number would be used. + * Slap them together if this is the case. + */ + if (ci->family == 6 || ci->family == 15) { + ext_model = (eax >> 16) & 0xF; + ci->model |= (ext_model << 4); + } +} + +void +cpu_shootdown_tlb(vaddr_t va) +{ + uint32_t ncpu = cpu_count(); + struct cpu_info *cip; + + for (uint32_t i = 0; i < ncpu; ++i) { + cip = cpu_get(i); + if (cip == NULL) { + break; + } + + spinlock_acquire(&cip->lock); + cip->shootdown_va = va; + cip->tlb_shootdown = 1; + lapic_send_ipi(cip->apicid, IPI_SHORTHAND_NONE, TLB_VECTOR); + spinlock_release(&cip->lock); + } +} + void md_backtrace(void) { @@ -138,6 +254,7 @@ md_backtrace(void) uintptr_t rip; off_t off; const char *name; + char line[256]; __ASMV("mov %%rbp, %0" : "=r" (rbp) :: "memory"); while (1) { @@ -150,7 +267,8 @@ md_backtrace(void) if (name == NULL) name = "???"; - kprintf(OMIT_TIMESTAMP "%p @ <%s+0x%x>\n", rip, name, off); + snprintf(line, sizeof(line), "%p @ <%s+0x%x>\n", rip, name, off); + cons_putstr(&g_root_scr, line, strlen(line)); } } @@ -168,10 +286,25 @@ cpu_halt_all(void) } /* Send IPI to all cores */ - lapic_send_ipi(0, IPI_SHORTHAND_ALL, halt_vector); + lapic_send_ipi(0, IPI_SHORTHAND_ALL, HALT_VECTOR); for (;;); } +/* + * Same as cpu_halt_all() but for all other + * cores but ourselves. + */ +void +cpu_halt_others(void) +{ + if (rdmsr(IA32_GS_BASE) == 0) { + __ASMV("cli; hlt"); + } + + /* Send IPI to all cores */ + lapic_send_ipi(0, IPI_SHORTHAND_OTHERS, HALT_VECTOR); +} + void serial_init(void) { @@ -193,6 +326,10 @@ this_cpu(void) { struct cpu_info *ci; + if (rdmsr(IA32_GS_BASE) == 0) { + return NULL; + } + /* * This might look crazy but we are just leveraging the "m" * constraint to add the offset of the self field within @@ -207,11 +344,65 @@ this_cpu(void) return ci; } +/* + * Sync all system operation + */ +int +md_sync_all(void) +{ + lapic_eoi(); + i8042_sync(); + return 0; +} + +void +cpu_enable_smep(void) +{ + struct cpu_info *ci; + uint64_t cr4; + + /* Don't bother if not enabled */ + if (!CPU_SMEP) { + return; + } + + ci = this_cpu(); + if (!ISSET(ci->feat, CPU_FEAT_SMEP)) { + pr_trace_bsp("SMEP not supported\n"); + return; + } + + cr4 = amd64_read_cr4(); + cr4 |= BIT(20); /* CR4.SMEP */ + amd64_write_cr4(cr4); +} + +void +cpu_disable_smep(void) +{ + struct cpu_info *ci; + uint64_t cr4; + + if (!CPU_SMEP) { + return; + } + + ci = this_cpu(); + if (!ISSET(ci->feat, CPU_FEAT_SMEP)) { + return; + } + + cr4 = amd64_read_cr4(); + cr4 &= ~BIT(20); /* CR4.SMEP */ + amd64_write_cr4(cr4); +} + void cpu_startup(struct cpu_info *ci) { ci->self = ci; - gdt_load(&bsp_gdtr); + ci->feat = 0; + gdt_load(); idt_load(); setup_vectors(); @@ -220,6 +411,13 @@ cpu_startup(struct cpu_info *ci) init_tss(ci); try_mitigate_spectre(); - __ASMV("sti"); /* Unmask interrupts */ + cpu_get_info(ci); + cpu_enable_smep(); + + enable_simd(); lapic_init(); + + if (!bsp_init) { + bsp_init = true; + } } |