summaryrefslogtreecommitdiff
path: root/sys/arch
diff options
context:
space:
mode:
Diffstat (limited to 'sys/arch')
-rw-r--r--sys/arch/aarch64/aarch64/pmap.c21
-rw-r--r--sys/arch/amd64/amd64/intr.c7
-rw-r--r--sys/arch/amd64/amd64/machdep.c52
-rw-r--r--sys/arch/amd64/amd64/mp.c28
-rw-r--r--sys/arch/amd64/amd64/pmap.c36
-rw-r--r--sys/arch/amd64/amd64/proc_machdep.c23
-rw-r--r--sys/arch/amd64/amd64/vector.S126
7 files changed, 217 insertions, 76 deletions
diff --git a/sys/arch/aarch64/aarch64/pmap.c b/sys/arch/aarch64/aarch64/pmap.c
index b5ebda9..7e93959 100644
--- a/sys/arch/aarch64/aarch64/pmap.c
+++ b/sys/arch/aarch64/aarch64/pmap.c
@@ -66,3 +66,24 @@ pmap_destroy_vas(struct vas vas)
/* TODO: STUB */
return;
}
+
+bool
+pmap_is_clean(struct vas vas, vaddr_t va)
+{
+ /* TODO: STUB */
+ return false;
+}
+
+void
+pmap_mark_clean(struct vas vas, vaddr_t va)
+{
+ /* TODO: STUB */
+ return;
+}
+
+int
+pmap_set_cache(struct vas vas, vaddr_t va, int type)
+{
+ /* TODO: STUB */
+ return 0;
+}
diff --git a/sys/arch/amd64/amd64/intr.c b/sys/arch/amd64/amd64/intr.c
index 1529b1c..9f70e88 100644
--- a/sys/arch/amd64/amd64/intr.c
+++ b/sys/arch/amd64/amd64/intr.c
@@ -96,11 +96,12 @@ intr_register(const char *name, const struct intr_hand *ih)
* Try to allocate an interrupt vector. An IPL is made up
* of 4 bits so there can be 16 vectors per IPL.
*
- * XXX: Vector 0x20 is reserved for the Hyra scheduler and
- * vector 0x21 is reserved for the CPU halt IPI.
+ * XXX: Vector 0x20 is reserved for the Hyra scheduler,
+ * vector 0x21 is reserved for the CPU halt IPI,
+ * and vector 0x22 is reserved for TLB shootdowns.
*/
for (int i = vec; i < vec + 16; ++i) {
- if (g_intrs[i] != NULL || i < 0x22) {
+ if (g_intrs[i] != NULL || i < 0x23) {
continue;
}
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index 3381437..c6fb6c4 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -42,9 +42,11 @@
#include <machine/uart.h>
#include <machine/sync.h>
#include <machine/intr.h>
+#include <machine/cdefs.h>
#include <machine/isa/i8042var.h>
#define HALT_VECTOR 0x21
+#define TLB_VECTOR 0x22
#if defined(__SPECTRE_IBRS)
#define SPECTRE_IBRS __SPECTRE_IBRS
@@ -52,8 +54,6 @@
#define SPECTRE_IBRS 0
#endif
-static uint8_t halt_vector = 0;
-
int ibrs_enable(void);
void syscall_isr(void);
void pin_isr_load(void);
@@ -68,6 +68,31 @@ cpu_halt_isr(void *p)
__builtin_unreachable();
}
+__attribute__((__interrupt__))
+static void
+tlb_shootdown_isr(void *p)
+{
+ struct cpu_info *ci;
+ int ipl;
+
+ /*
+ * Get the current CPU and check if we even
+ * need a shootdown. If `tlb_shootdown' is
+ * unset, this is not for us.
+ */
+ ci = this_cpu();
+ if (!ci->tlb_shootdown) {
+ return;
+ }
+
+ ipl = splraise(IPL_HIGH);
+ __invlpg(ci->shootdown_va);
+
+ ci->shootdown_va = 0;
+ ci->tlb_shootdown = 0;
+ splx(ipl);
+}
+
static void
setup_vectors(void)
{
@@ -85,6 +110,7 @@ setup_vectors(void)
idt_set_desc(0xE, IDT_TRAP_GATE, ISR(page_fault), 0);
idt_set_desc(0x80, IDT_USER_INT_GATE, ISR(syscall_isr), 0);
idt_set_desc(HALT_VECTOR, IDT_INT_GATE, ISR(cpu_halt_isr), 0);
+ idt_set_desc(TLB_VECTOR, IDT_INT_GATE, ISR(tlb_shootdown_isr), 0);
pin_isr_load();
}
@@ -130,6 +156,26 @@ backtrace_addr_to_name(uintptr_t addr, off_t *off)
}
void
+cpu_shootdown_tlb(vaddr_t va)
+{
+ uint32_t ncpu = cpu_count();
+ struct cpu_info *cip;
+
+ for (uint32_t i = 0; i < ncpu; ++i) {
+ cip = cpu_get(i);
+ if (cip == NULL) {
+ break;
+ }
+
+ spinlock_acquire(&cip->lock);
+ cip->shootdown_va = va;
+ cip->tlb_shootdown = 1;
+ lapic_send_ipi(cip->apicid, IPI_SHORTHAND_NONE, TLB_VECTOR);
+ spinlock_release(&cip->lock);
+ }
+}
+
+void
md_backtrace(void)
{
uintptr_t *rbp;
@@ -166,7 +212,7 @@ cpu_halt_all(void)
}
/* Send IPI to all cores */
- lapic_send_ipi(0, IPI_SHORTHAND_ALL, halt_vector);
+ lapic_send_ipi(0, IPI_SHORTHAND_ALL, HALT_VECTOR);
for (;;);
}
diff --git a/sys/arch/amd64/amd64/mp.c b/sys/arch/amd64/amd64/mp.c
index 22561d7..1e6d8d9 100644
--- a/sys/arch/amd64/amd64/mp.c
+++ b/sys/arch/amd64/amd64/mp.c
@@ -29,6 +29,7 @@
#include <sys/types.h>
#include <sys/limine.h>
+#include <sys/limits.h>
#include <sys/syslog.h>
#include <sys/spinlock.h>
#include <sys/sched.h>
@@ -45,7 +46,9 @@ static volatile struct limine_smp_request g_smp_req = {
.revision = 0
};
-static volatile uint32_t ncpu_up = 0;
+static volatile uint32_t ncpu_up = 1;
+static struct cpu_info *ci_list[CPU_MAX];
+static struct spinlock ci_list_lock = {0};
static void
ap_trampoline(struct limine_smp_info *si)
@@ -57,11 +60,31 @@ ap_trampoline(struct limine_smp_info *si)
memset(ci, 0, sizeof(*ci));
cpu_startup(ci);
+ spinlock_acquire(&ci_list_lock);
+ ci_list[ncpu_up] = ci;
+ spinlock_release(&ci_list_lock);
+
atomic_inc_int(&ncpu_up);
sched_enter();
while (1);
}
+struct cpu_info *
+cpu_get(uint32_t index)
+{
+ if (index >= ncpu_up) {
+ return NULL;
+ }
+
+ return ci_list[index];
+}
+
+uint32_t
+cpu_count(void)
+{
+ return ncpu_up;
+}
+
void
mp_bootstrap_aps(struct cpu_info *ci)
{
@@ -74,6 +97,7 @@ mp_bootstrap_aps(struct cpu_info *ci)
cpus = resp->cpus;
cpu_init_counter = resp->cpu_count - 1;
+ ci_list[0] = ci;
if (resp->cpu_count == 1) {
pr_trace("CPU has 1 core, no APs to bootstrap...\n");
@@ -91,5 +115,5 @@ mp_bootstrap_aps(struct cpu_info *ci)
}
/* Wait for all cores to be ready */
- while (ncpu_up < cpu_init_counter);
+ while ((ncpu_up - 1) < cpu_init_counter);
}
diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c
index 2e62a4b..0bdf3b7 100644
--- a/sys/arch/amd64/amd64/pmap.c
+++ b/sys/arch/amd64/amd64/pmap.c
@@ -33,6 +33,8 @@
#include <sys/errno.h>
#include <machine/tlb.h>
#include <machine/vas.h>
+#include <machine/cpu.h>
+#include <machine/cdefs.h>
#include <vm/pmap.h>
#include <vm/physmem.h>
#include <vm/vm.h>
@@ -303,3 +305,37 @@ pmap_set_cache(struct vas vas, vaddr_t va, int type)
return 0;
}
+
+bool
+pmap_is_clean(struct vas vas, vaddr_t va)
+{
+ uintptr_t *tbl;
+ int status;
+ size_t idx;
+
+ if ((status = pmap_get_tbl(vas, va, false, &tbl)) != 0)
+ return status;
+
+ idx = pmap_get_level_index(1, va);
+ return ISSET(tbl[idx], PTE_DIRTY) == 0;
+}
+
+void
+pmap_mark_clean(struct vas vas, vaddr_t va)
+{
+ uintptr_t *tbl;
+ int status;
+ size_t idx;
+
+ if ((status = pmap_get_tbl(vas, va, false, &tbl)) != 0)
+ return;
+
+ idx = pmap_get_level_index(1, va);
+ tbl[idx] &= ~PTE_DIRTY;
+
+ if (cpu_count() > 1) {
+ cpu_shootdown_tlb(va);
+ } else {
+ __invlpg(va);
+ }
+}
diff --git a/sys/arch/amd64/amd64/proc_machdep.c b/sys/arch/amd64/amd64/proc_machdep.c
index f6b91f9..4fe8580 100644
--- a/sys/arch/amd64/amd64/proc_machdep.c
+++ b/sys/arch/amd64/amd64/proc_machdep.c
@@ -123,24 +123,37 @@ md_td_kick(struct proc *td)
{
struct trapframe *tfp;
struct cpu_info *ci;
+ uint8_t rpl;
+ uint16_t ds = KERNEL_DS;
tfp = &td->tf;
+ rpl = tfp->cs & 3;
ci = this_cpu();
ci->curtd = td;
+ if (rpl == 3) {
+ td->flags &= ~PROC_KTD;
+ ds = USER_DS | 3;
+ }
+
__ASMV(
- "push %0\n"
+ "mov %0, %%rax\n"
"push %1\n"
- "pushf\n"
"push %2\n"
"push %3\n"
+ "push %%rax\n"
+ "push %4\n"
+ "test $3, %%ax\n"
+ "jz 1f\n"
"lfence\n"
"swapgs\n"
- "iretq"
+ "1:\n"
+ " iretq"
:
- : "i" (USER_DS | 3),
+ : "r" (tfp->cs),
+ "r" (ds),
"r" (tfp->rsp),
- "i" (USER_CS | 3),
+ "m" (tfp->rflags),
"r" (tfp->rip)
);
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index 32ccd34..d2a3d89 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -64,69 +64,69 @@ done:
.globl pin_isr_load
pin_isr_load:
- IDT_SET_VEC 34, ioapic_edge_0
- IDT_SET_VEC 35, ioapic_edge_1
- IDT_SET_VEC 36, ioapic_edge_2
- IDT_SET_VEC 37, ioapic_edge_3
- IDT_SET_VEC 38, ioapic_edge_4
- IDT_SET_VEC 39, ioapic_edge_5
- IDT_SET_VEC 40, ioapic_edge_6
- IDT_SET_VEC 41, ioapic_edge_7
- IDT_SET_VEC 42, ioapic_edge_8
- IDT_SET_VEC 43, ioapic_edge_9
- IDT_SET_VEC 44, ioapic_edge_10
- IDT_SET_VEC 45, ioapic_edge_11
- IDT_SET_VEC 46, ioapic_edge_12
- IDT_SET_VEC 47, ioapic_edge_13
- IDT_SET_VEC 48, ioapic_edge_14
- IDT_SET_VEC 49, ioapic_edge_15
- IDT_SET_VEC 50, ioapic_edge_16
- IDT_SET_VEC 51, ioapic_edge_17
- IDT_SET_VEC 52, ioapic_edge_18
- IDT_SET_VEC 53, ioapic_edge_19
- IDT_SET_VEC 54, ioapic_edge_20
- IDT_SET_VEC 55, ioapic_edge_21
- IDT_SET_VEC 56, ioapic_edge_22
- IDT_SET_VEC 57, ioapic_edge_23
- IDT_SET_VEC 58, ioapic_edge_24
- IDT_SET_VEC 59, ioapic_edge_25
- IDT_SET_VEC 60, ioapic_edge_26
- IDT_SET_VEC 61, ioapic_edge_27
- IDT_SET_VEC 62, ioapic_edge_28
- IDT_SET_VEC 63, ioapic_edge_29
- IDT_SET_VEC 64, ioapic_edge_30
- IDT_SET_VEC 65, ioapic_edge_31
- IDT_SET_VEC 66, ioapic_edge_32
- IDT_SET_VEC 67, ioapic_edge_33
- IDT_SET_VEC 68, ioapic_edge_34
- IDT_SET_VEC 69, ioapic_edge_35
- IDT_SET_VEC 70, ioapic_edge_36
- IDT_SET_VEC 71, ioapic_edge_37
- IDT_SET_VEC 72, ioapic_edge_38
- IDT_SET_VEC 73, ioapic_edge_39
- IDT_SET_VEC 74, ioapic_edge_40
- IDT_SET_VEC 75, ioapic_edge_41
- IDT_SET_VEC 76, ioapic_edge_42
- IDT_SET_VEC 77, ioapic_edge_43
- IDT_SET_VEC 78, ioapic_edge_44
- IDT_SET_VEC 79, ioapic_edge_45
- IDT_SET_VEC 80, ioapic_edge_46
- IDT_SET_VEC 81, ioapic_edge_47
- IDT_SET_VEC 82, ioapic_edge_48
- IDT_SET_VEC 83, ioapic_edge_49
- IDT_SET_VEC 84, ioapic_edge_50
- IDT_SET_VEC 85, ioapic_edge_51
- IDT_SET_VEC 86, ioapic_edge_52
- IDT_SET_VEC 87, ioapic_edge_53
- IDT_SET_VEC 88, ioapic_edge_54
- IDT_SET_VEC 89, ioapic_edge_55
- IDT_SET_VEC 90, ioapic_edge_56
- IDT_SET_VEC 91, ioapic_edge_57
- IDT_SET_VEC 92, ioapic_edge_58
- IDT_SET_VEC 93, ioapic_edge_59
- IDT_SET_VEC 94, ioapic_edge_60
- IDT_SET_VEC 95, ioapic_edge_61
- IDT_SET_VEC 96, ioapic_edge_62
+ IDT_SET_VEC 35, ioapic_edge_0
+ IDT_SET_VEC 36, ioapic_edge_1
+ IDT_SET_VEC 37, ioapic_edge_2
+ IDT_SET_VEC 38, ioapic_edge_3
+ IDT_SET_VEC 39, ioapic_edge_4
+ IDT_SET_VEC 40, ioapic_edge_5
+ IDT_SET_VEC 41, ioapic_edge_6
+ IDT_SET_VEC 42, ioapic_edge_7
+ IDT_SET_VEC 43, ioapic_edge_8
+ IDT_SET_VEC 44, ioapic_edge_9
+ IDT_SET_VEC 45, ioapic_edge_10
+ IDT_SET_VEC 46, ioapic_edge_11
+ IDT_SET_VEC 47, ioapic_edge_12
+ IDT_SET_VEC 48, ioapic_edge_13
+ IDT_SET_VEC 49, ioapic_edge_14
+ IDT_SET_VEC 50, ioapic_edge_15
+ IDT_SET_VEC 51, ioapic_edge_16
+ IDT_SET_VEC 52, ioapic_edge_17
+ IDT_SET_VEC 53, ioapic_edge_18
+ IDT_SET_VEC 54, ioapic_edge_19
+ IDT_SET_VEC 55, ioapic_edge_20
+ IDT_SET_VEC 56, ioapic_edge_21
+ IDT_SET_VEC 57, ioapic_edge_22
+ IDT_SET_VEC 58, ioapic_edge_23
+ IDT_SET_VEC 59, ioapic_edge_24
+ IDT_SET_VEC 60, ioapic_edge_25
+ IDT_SET_VEC 61, ioapic_edge_26
+ IDT_SET_VEC 62, ioapic_edge_27
+ IDT_SET_VEC 63, ioapic_edge_28
+ IDT_SET_VEC 64, ioapic_edge_29
+ IDT_SET_VEC 65, ioapic_edge_30
+ IDT_SET_VEC 66, ioapic_edge_31
+ IDT_SET_VEC 67, ioapic_edge_32
+ IDT_SET_VEC 68, ioapic_edge_33
+ IDT_SET_VEC 69, ioapic_edge_34
+ IDT_SET_VEC 70, ioapic_edge_35
+ IDT_SET_VEC 71, ioapic_edge_36
+ IDT_SET_VEC 72, ioapic_edge_37
+ IDT_SET_VEC 73, ioapic_edge_38
+ IDT_SET_VEC 74, ioapic_edge_39
+ IDT_SET_VEC 75, ioapic_edge_40
+ IDT_SET_VEC 76, ioapic_edge_41
+ IDT_SET_VEC 77, ioapic_edge_42
+ IDT_SET_VEC 78, ioapic_edge_43
+ IDT_SET_VEC 79, ioapic_edge_44
+ IDT_SET_VEC 80, ioapic_edge_45
+ IDT_SET_VEC 81, ioapic_edge_46
+ IDT_SET_VEC 82, ioapic_edge_47
+ IDT_SET_VEC 83, ioapic_edge_48
+ IDT_SET_VEC 84, ioapic_edge_49
+ IDT_SET_VEC 85, ioapic_edge_50
+ IDT_SET_VEC 86, ioapic_edge_51
+ IDT_SET_VEC 87, ioapic_edge_52
+ IDT_SET_VEC 88, ioapic_edge_53
+ IDT_SET_VEC 89, ioapic_edge_54
+ IDT_SET_VEC 90, ioapic_edge_55
+ IDT_SET_VEC 91, ioapic_edge_56
+ IDT_SET_VEC 92, ioapic_edge_57
+ IDT_SET_VEC 93, ioapic_edge_58
+ IDT_SET_VEC 94, ioapic_edge_59
+ IDT_SET_VEC 95, ioapic_edge_60
+ IDT_SET_VEC 96, ioapic_edge_61
+ IDT_SET_VEC 97, ioapic_edge_62
IDT_SET_VEC 97, ioapic_edge_63
ret
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-02 09:14:35 +0000