summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/arch/amd64/amd64/ipi.c271
-rw-r--r--sys/arch/amd64/amd64/machdep.c16
-rw-r--r--sys/arch/amd64/amd64/vector.S24
-rw-r--r--sys/include/arch/amd64/cpu.h5
-rw-r--r--sys/include/arch/amd64/ipi.h51
5 files changed, 62 insertions, 305 deletions
diff --git a/sys/arch/amd64/amd64/ipi.c b/sys/arch/amd64/amd64/ipi.c
index ffa291f..e367552 100644
--- a/sys/arch/amd64/amd64/ipi.c
+++ b/sys/arch/amd64/amd64/ipi.c
@@ -37,69 +37,28 @@
#include <machine/idt.h>
#include <machine/ipi.h>
#include <machine/lapic.h>
+#include <vm/dynalloc.h>
#include <string.h>
-void ipi_isr0(void);
-void ipi_isr1(void);
-void ipi_isr2(void);
-void ipi_isr3(void);
+void ipi_isr(void);
void __ipi_handle_common(void);
#define pr_trace(fmt, ...) kprintf("ipi: " fmt, ##__VA_ARGS__)
#define pr_error(...) pr_trace(__VA_ARGS__)
-#define BASE_VECTOR 0x21
#define COOKIE 0x7E0A
+#define IPI_VECTOR 0x21
+#define MAX_IPI 32
/* For the global state of the subsystem */
static uint32_t cookie = 0;
-/*
- * The next vector that will be used for an IPI to
- * be allocated. It starts at 0x21 because interrupt
- * vector 0x20 is used for the Hyra scheduler and `N_IPIVEC'
- * vectors up are reserved for inter-processor interrupts.
- *
- * XXX: This must not go beyond N_IPIVEC !!
- */
-static uint8_t next_vec = BASE_VECTOR;
-static uint8_t vec_entries = 0;
-
-/*
- * In order to get an index into the 'vectors' array,
- * one can pass an `ipi_bitmap' bit index into the
- * ipi_vector() function. The index into the `ipi`
- * field within may be acquired with the ipi_index()
- * function.
- */
-static uint64_t ipi_bitmap = 0;
-static struct ipi_vector vectors[N_IPIVEC];
+static struct cpu_ipi ipi_list[MAX_IPI];
+static uint8_t ipi_count = 0;
static struct spinlock lock;
/*
- * Allocate a bit from the `ipi_bitmap' and
- * return the index.
- *
- * Returns a less than zero value upon error.
- */
-static ssize_t
-alloc_ipi_bit(void)
-{
- const size_t MAX = sizeof(ipi_bitmap) * 8;
- off_t i;
-
- for (i = 0; i < MAX; ++i) {
- if (!ISSET(ipi_bitmap, BIT(i))) {
- ipi_bitmap |= BIT(i);
- return i;
- }
- }
-
- return -1;
-}
-
-/*
* Allocate an IPI that can be sent to other
* cores on the CPU. This is the core logic
* and contains *no* locks. One should be
@@ -111,167 +70,18 @@ alloc_ipi_bit(void)
static int
__ipi_alloc(struct cpu_ipi **res)
{
- struct ipi_vector *vp;
struct cpu_ipi *ipip;
- ssize_t bit;
- uint8_t idx;
-
- if (res == NULL) {
- return -EINVAL;
- }
- if (next_vec >= BASE_VECTOR + N_IPIVEC) {
+ if (ipi_count >= MAX_IPI) {
return -EAGAIN;
}
- /*
- * Attempt to allocate a bit index from
- * the bitmap.
- */
- if ((bit = alloc_ipi_bit()) < 0) {
- return -EAGAIN;
- }
-
- idx = ipi_vector(bit);
- vp = &vectors[idx];
-
- /* Initialize the vector if not already */
- if (vp->cookie != COOKIE) {
- vp->cookie = COOKIE;
- vp->nipi = 0;
- vp->vec = next_vec;
- memset(vp->ipi, 0, sizeof(vp->ipi));
- }
-
- /*
- * Just a sanity check here, the number of
- * IPIs per vector should never exceed the
- * maximum, and if it does, that gives us more
- * than enough grounds to panic the system as
- * it would not be wise to trust it.
- */
- if (__unlikely(vp->nipi >= IPI_PER_VEC)) {
- panic("too many IPIs in vector %x\n", vp->vec);
- }
-
- idx = ipi_index(bit);
- ipip = &vp->ipi[idx];
-
- /* We are allocating, not clobbering */
- if (ipip->cookie == COOKIE) {
- panic("ipi table corruption\n");
- }
-
- if ((++vec_entries) >= IPI_PER_VEC) {
- vec_entries = 0;
- ++next_vec;
- }
-
- /* Set up the initial state */
+ ipip = &ipi_list[ipi_count];
ipip->cookie = COOKIE;
+ ipip->id = ipi_count++;
ipip->handler = NULL;
- ipip->id = bit;
*res = ipip;
- return bit;
-}
-
-/*
- * Dispatch pending IPIs for the current
- * processor.
- *
- * @vector: Backing interrupt vector
- * @ci: Current processor
- */
-static void
-ipi_dispatch_pending(struct ipi_vector *vec, struct cpu_info *ci)
-{
- uint8_t bit_i;
- uint8_t n_bit;
- uint8_t index;
- struct cpu_ipi *ipip = NULL;
- ipi_pend_t pending;
-
- if (vec == NULL || ci == NULL) {
- return;
- }
-
- n_bit = sizeof(pending) * 8;
- for (bit_i = 0; bit_i < n_bit; ++bit_i) {
- index = ipi_vector(bit_i);
- pending = ci->ipi_pending[index];
-
- vec = &vectors[index];
- index = ipi_index(bit_i);
- ipip = &vec->ipi[index];
-
- /* Is this pending or not? */
- if (!ISSET(pending, BIT(bit_i))) {
- continue;
- }
-
- /* Handle and mark as no longer pending */
- ipip->handler(ipip);
- ci->ipi_pending[vec->vec] &= ~BIT(bit_i);
- }
-}
-
-/*
- * Check an IPI pending bitmap for a
- * vector and send IPIs as needed
- *
- * @ci: Target processor
- * @pending: Pending IPIs
- */
-static void
-ipi_send_vector(struct cpu_info *ci, ipi_pend_t pending)
-{
- struct ipi_vector *vp;
- struct cpu_ipi *ipip;
- uint8_t n_bits = sizeof(pending) * 8;
- uint8_t bit_i;
- uint8_t vector, index;
- uint32_t apic_id = 0;
-
- if (ci != NULL) {
- /*
- * We are already dispatching IPIs, we don't
- * want to find ourselves in interrupt hell.
- */
- if (ci->ipi_dispatch) {
- return;
- }
-
- apic_id = ci->apicid;
- }
-
- ci->ipi_dispatch = 1;
- for (bit_i = 0; bit_i < n_bits; ++bit_i) {
- if (ISSET(pending, BIT(bit_i))) {
- vector = ipi_vector(bit_i);
- index = ipi_index(bit_i);
-
- if (ci != NULL)
- ci->ipi_id = bit_i;
-
- vp = &vectors[vector];
- ipip = &vp->ipi[index];
-
- /* Ignore if cookie does match */
- if (ipip->cookie != COOKIE)
- continue;
-
- /* Ignore if there is no handler */
- if (ipip->handler == NULL)
- continue;
-
- /* Send that IPI through */
- lapic_send_ipi(
- apic_id,
- IPI_SHORTHAND_NONE,
- BASE_VECTOR + vector
- );
- }
- }
+ return ipip->id;
}
/*
@@ -282,33 +92,37 @@ ipi_send_vector(struct cpu_info *ci, ipi_pend_t pending)
void
__ipi_handle_common(void)
{
- struct ipi_vector *vp;
+ struct cpu_ipi *ipip;
struct cpu_info *ci = this_cpu();
- uint8_t vector;
+ ipi_pend_t pending = 0;
if (cookie != COOKIE) {
pr_trace("[warn]: got spurious ipi\n");
return;
}
- /* Grab the vector */
- vector = ipi_vector(ci->ipi_id);
- vp = &vectors[vector];
- if (vp->cookie != COOKIE) {
- pr_error("got IPI for uninitialized vector\n");
+ if (ci == NULL) {
+ pr_error("could not get current CPU\n");
return;
}
- if ((ci = this_cpu()) == NULL) {
- pr_error("could not get current CPU\n");
+ if (ipi_count == 0) {
+ pr_error("no registered IPIs\n");
return;
}
- ipi_dispatch_pending(vp, ci);
+ /* Attempt to find a handler */
+ pending = ci->ipi_pending;
+ for (int i = 0; i < ipi_count; ++i) {
+ ipip = &ipi_list[i];
+ if (ISSET(pending, BIT(i))) {
+ ipip->handler(ipip);
+ ci->ipi_pending &= ~BIT(i);
+ }
+ }
/* We are done dispatching IPIs */
ci->ipi_dispatch = 0;
- ci->ipi_id = 0;
}
/*
@@ -317,20 +131,34 @@ __ipi_handle_common(void)
* the `ci->ipi_pending' field
*
* @ci: Processor to send IPI(s) to
+ * @ipi: IPIs to send
*/
int
-md_ipi_send(struct cpu_info *ci)
+md_ipi_send(struct cpu_info *ci, ipi_pend_t ipi)
{
- if (ci == NULL) {
- return -EINVAL;
- }
+ uint32_t apic_id = 0;
- spinlock_acquire(&lock);
- for (int i = 0; i < N_IPIVEC; ++i) {
- ipi_send_vector(ci, ci->ipi_pending[i]);
+ if (ci != NULL) {
+ /*
+ * We are already dispatching IPIs, we don't
+ * want to find ourselves in interrupt hell.
+ */
+ if (ci->ipi_dispatch) {
+ return -EAGAIN;
+ }
+
+ apic_id = ci->apicid;
}
- spinlock_release(&lock);
+ ci->ipi_dispatch = 1;
+ ci->ipi_pending |= BIT(ipi);
+
+ /* Send it through on the bus */
+ lapic_send_ipi(
+ apic_id,
+ IPI_SHORTHAND_NONE,
+ IPI_VECTOR
+ );
return 0;
}
@@ -357,9 +185,6 @@ void
md_ipi_init(void)
{
/* Initialize the IPI vectors */
- idt_set_desc(0x21, IDT_INT_GATE, ISR(ipi_isr0), 0);
- idt_set_desc(0x22, IDT_INT_GATE, ISR(ipi_isr1), 0);
- idt_set_desc(0x23, IDT_INT_GATE, ISR(ipi_isr2), 0);
- idt_set_desc(0x24, IDT_INT_GATE, ISR(ipi_isr3), 0);
+ idt_set_desc(IPI_VECTOR, IDT_INT_GATE, ISR(ipi_isr), 0);
cookie = COOKIE;
}
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index 5fb006c..2976a51 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -404,7 +404,7 @@ cpu_shootdown_tlb(vaddr_t va)
spinlock_acquire(&cip->lock);
cip->shootdown_va = va;
cip->tlb_shootdown = 1;
- cpu_ipi_send(cip, IPI_TLB);
+ md_ipi_send(cip, IPI_TLB);
spinlock_release(&cip->lock);
}
}
@@ -461,7 +461,7 @@ md_backtrace(void)
void
cpu_halt_all(void)
{
- struct cpu_info *ci;
+ struct cpu_info *ci, *curcpu;
uint32_t ncpu = cpu_count();
/*
@@ -470,19 +470,21 @@ cpu_halt_all(void)
* processor is the only one active, clear interrupts
* then halt it.
*/
- if (rdmsr(IA32_GS_BASE) == 0) {
- __ASMV("cli; hlt");
+ __ASMV("cli");
+ if ((curcpu = this_cpu()) == NULL) {
+ __ASMV("hlt");
}
for (int i = 0; i < ncpu; ++i) {
ci = cpu_get(i);
- if (ci == NULL) {
+ if (ci->id == curcpu->id) {
continue;
}
- cpu_ipi_send(ci, IPI_HALT);
+ md_ipi_send(ci, IPI_HALT);
}
+ __ASMV("hlt");
for (;;);
}
@@ -508,7 +510,7 @@ cpu_halt_others(void)
if (ci->id == curcpu->id)
continue;
- cpu_ipi_send(ci, IPI_HALT);
+ md_ipi_send(ci, IPI_HALT);
}
}
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index 19c68d5..69173d7 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -139,31 +139,11 @@ pin_isr_load:
IDT_SET_VEC 100, ioapic_edge_63
ret
- .globl ipi_isr0
-INTRENTRY(ipi_isr0, ipi_trampoline)
+ .globl ipi_isr
+INTRENTRY(ipi_isr, ipi_trampoline)
call ipi_trampoline
retq
- .globl ipi_isr1
-INTRENTRY(ipi_isr1, ipi_trampoline)
- call ipi_trampoline
- retq
-
- .globl ipi_isr2
-INTRENTRY(ipi_isr2, ipi_trampoline)
- call ipi_trampoline
- retq
-
- .globl ipi_isr3
-INTRENTRY(ipi_isr3, ipi_trampoline)
- call ipi_trampoline
- retq
-
-/*
- * Hyra supports 16 IPI handlers per 4 reserved
- * IDT vectors. That allows for a maximum of
- * 64 IPIs.
- */
ipi_trampoline:
call __ipi_handle_common
retq
diff --git a/sys/include/arch/amd64/cpu.h b/sys/include/arch/amd64/cpu.h
index cf073fe..5adff29 100644
--- a/sys/include/arch/amd64/cpu.h
+++ b/sys/include/arch/amd64/cpu.h
@@ -52,7 +52,7 @@
#define CPU_VENDOR_INTEL 0x00000001
#define CPU_VENDOR_AMD 0x00000002
-typedef uint16_t ipi_pend_t;
+typedef uint32_t ipi_pend_t;
struct cpu_info {
uint32_t apicid;
@@ -60,8 +60,7 @@ struct cpu_info {
uint32_t vendor; /* Vendor (see CPU_VENDOR_*) */
uint8_t preempt : 1; /* CPU is preemptable */
uint8_t ipi_dispatch : 1; /* 1: IPIs being dispatched */
- uint8_t ipi_id;
- ipi_pend_t ipi_pending[N_IPIVEC];
+ ipi_pend_t ipi_pending;
uint8_t id; /* MI Logical ID */
uint8_t model : 4; /* CPU model number */
uint8_t family : 4; /* CPU family ID */
diff --git a/sys/include/arch/amd64/ipi.h b/sys/include/arch/amd64/ipi.h
index 1a3b51c..452d006 100644
--- a/sys/include/arch/amd64/ipi.h
+++ b/sys/include/arch/amd64/ipi.h
@@ -53,57 +53,8 @@ struct cpu_ipi {
int(*handler)(struct cpu_ipi *ipi);
};
-/*
- * Represents an interrupt vector for a
- * specific IPI
- *
- * @ipi: IPIs associated with this vector
- * @cookie: Used to verify an instance
- * @nipi: Number of IPIs associated
- * @vec: System interrupt vector number
- */
-struct ipi_vector {
- struct cpu_ipi ipi[IPI_PER_VEC];
- uint16_t cookie;
- uint8_t nipi;
- uint8_t vec;
-};
-
int md_ipi_alloc(struct cpu_ipi **res);
-int md_ipi_send(struct cpu_info *ci);
+int md_ipi_send(struct cpu_info *ci, ipi_pend_t ipi);
void md_ipi_init(void);
-/*
- * Get the vector an IPI belongs to
- *
- * @ipi: IPI to check
- */
-__always_inline static inline uint8_t
-ipi_vector(uint8_t ipi)
-{
- return ipi / N_IPIVEC;
-}
-
-/*
- * Get the handler index an IPI belongs
- * to
- *
- * @ipi: IPI to check
- */
-__always_inline static inline uint8_t
-ipi_index(uint8_t ipi)
-{
- return ipi % (sizeof(ipi_pend_t) * 8);
-}
-
-__always_inline static inline int
-cpu_ipi_send(struct cpu_info *ci, uint8_t ipi)
-{
- uint8_t vec = ipi_vector(ipi);
- uint8_t idx = ipi_index(ipi);
-
- ci->ipi_pending[vec] |= BIT(idx);
- return md_ipi_send(ci);
-}
-
#endif /* !_MACHINE_IPI_H_ */
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-22 17:26:45 +0000