From a9d1f332d91a801ff04e2b9f2475d733b2898134 Mon Sep 17 00:00:00 2001
From: Ian Moffett <ian@osmora.org>
Date: Mon, 17 Nov 2025 21:15:55 -0500
Subject: kern/amd64: mp: Make AP bring up more stable

- Move BUDA to 0x9000
- Move the BUA to 0x8000
- Serialize bring up with is_booted flag
- Map whole 2 megs of lower address space

Signed-off-by: Ian Moffett <ian@osmora.org>
---
 sys/arch/amd64/boot/apboot.asm | 12 +++++++-----
 sys/arch/amd64/cpu/mp.c        | 12 ++++++++----
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/sys/arch/amd64/boot/apboot.asm b/sys/arch/amd64/boot/apboot.asm
index 8c16319..1bae9aa 100644
--- a/sys/arch/amd64/boot/apboot.asm
+++ b/sys/arch/amd64/boot/apboot.asm
@@ -28,9 +28,9 @@
 ;;
 
 [bits 16]
-[org 0x1000]
+[org 0x8000]
 
-%define AP_BUDA     0x3000
+%define AP_BUDA     0x9000
 %define IA32_EFER   0xC0000080
 
 [bits 16]
@@ -39,13 +39,13 @@ _start:
     out 0x21, al                    ;; Disable master PIC
     out 0xA1, al                    ;; Disable slave PIC
 
-    mov eax, dword [AP_BUDA]        ;; BUDA.CR3 -> EAX
-    mov cr3, eax                    ;; EAX -> CR3
-
     mov eax, cr4                    ;; CR4 -> EAX
     or eax, 0xA0                    ;; Enable physical address extension + PGE
     mov cr4, eax                    ;; Write it back
 
+    mov eax, dword [AP_BUDA]        ;; BUDA.CR3 -> EAX
+    mov cr3, eax                    ;; EAX -> CR3
+
     mov ecx, IA32_EFER              ;; Read IA32_EFER
     rdmsr                           ;; -> EAX
     or eax, 0xD00                   ;; Set EFER.LME + defaults
@@ -80,6 +80,8 @@ thunk64:
     mov gs, ax
     mov rsp, qword [AP_BUDA + 0x08]
     mov rbx, qword [AP_BUDA + 0x10]
+    mov rax, 1
+    xchg qword [AP_BUDA + 0x18], rax
     cld
     jmp rbx
 
diff --git a/sys/arch/amd64/cpu/mp.c b/sys/arch/amd64/cpu/mp.c
index 247313a..5c45788 100644
--- a/sys/arch/amd64/cpu/mp.c
+++ b/sys/arch/amd64/cpu/mp.c
@@ -49,11 +49,11 @@
  * within a page and be no larger and no smaller.
  */
 #define AP_BUA_LEN   0x1000                     /* Bring up area length in bytes */
-#define AP_BUA_PADDR 0x1000                     /* Bring up area [physical] */
+#define AP_BUA_PADDR 0x8000                     /* Bring up area [physical] */
 #define AP_BUA_VADDR PHYS_TO_VIRT(AP_BUA_PADDR) /* Bring up area [virtual] */
 
 /* Bring up descriptor area */
-#define AP_BUDA_PADDR 0x3000
+#define AP_BUDA_PADDR 0x9000
 #define AP_BUDA_VADDR PHYS_TO_VIRT(AP_BUDA_PADDR)
 
 /*
@@ -74,6 +74,7 @@ struct __packed ap_buda {
     uint64_t cr3;               /* 0x00 */
     uint64_t rsp;               /* 0x08 */
     uint64_t lm_entry;          /* 0x10 */
+    uint64_t is_booted;         /* 0x18 */
 };
 
 /*
@@ -150,7 +151,7 @@ cpu_init_bootspace(struct ap_bootspace *bs)
     new_pml4[0] = bs->pml3 | 3;     /* P+RW */
     pml3[0] = bs->pml2     | 3;     /* P+RW */
     pml2[0] = bs->pml1     | 3;     /* P+RW */
-    for (uint8_t i = 0; i < 4; ++i) {
+    for (uint16_t i = 0; i < 256; ++i) {
         pml1[i] = (0x1000 * i) | 3; /* P+RW */
     }
     return 0;
@@ -244,7 +245,10 @@ cpu_lapic_cb(struct apic_header *h, size_t arg)
         hpet_msleep(2);
     }
 
-    dtrace("bootstrapping lapic %d\n", lapic->apic_id);
+    /* Wait until AP is booted */
+    while (!buda->is_booted);
+    buda->is_booted = 0;
+
     return -1;  /* Keep going */
 }
 
-- 
cgit v1.2.3