--- //depot/projects/smpng/sys/amd64/amd64/intr_machdep.c 2005/12/14 21:21:48 +++ //depot/user/jhb/intr/amd64/amd64/intr_machdep.c 2005/12/21 21:37:11 @@ -63,6 +63,12 @@ static struct intsrc *interrupt_sources[NUM_IO_INTS]; static struct mtx intr_table_lock; +#ifdef SMP +static int assign_cpu; + +static void intr_assign_next_cpu(struct intsrc *isrc); +#endif + static void intr_init(void *__dummy); static void intrcnt_setname(const char *name, int index); static void intrcnt_updatename(struct intsrc *is); @@ -93,6 +99,7 @@ } intrcnt_register(isrc); interrupt_sources[vector] = isrc; + isrc->is_enabled = 0; mtx_unlock_spin(&intr_table_lock); return (0); } @@ -118,7 +125,17 @@ intr_priority(flags), flags, cookiep); if (error == 0) { intrcnt_updatename(isrc); - isrc->is_pic->pic_enable_intr(isrc); + mtx_lock_spin(&intr_table_lock); + if (!isrc->is_enabled) { + isrc->is_enabled = 1; +#ifdef SMP + if (assign_cpu) + intr_assign_next_cpu(isrc); +#endif + mtx_unlock_spin(&intr_table_lock); + isrc->is_pic->pic_enable_intr(isrc); + } else + mtx_unlock_spin(&intr_table_lock); isrc->is_pic->pic_enable_source(isrc); } return (error); @@ -166,6 +183,7 @@ */ (*isrc->is_count)++; PCPU_LAZY_INC(cnt.v_intr); + (*PCPU_GET(intr_count))++; ie = isrc->is_event; @@ -311,6 +329,7 @@ intrcnt_setname("???", 0); intrcnt_index = 1; mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); + intrcnt_add("cpu0: intrs", PCPU_PTR(intr_count)); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -335,3 +354,76 @@ db_dump_intr_event((*isrc)->is_event, verbose); } #endif + +#ifdef SMP +/* + * Support for balancing interrupt sources across CPUs. For now we just + * allocate CPUs round-robin. + */ + +static u_int cpu_apic_ids[MAXCPU]; +static int current_cpu, num_cpus; + +static void +intr_assign_next_cpu(struct intsrc *isrc) +{ + struct pic *pic; + u_int apic_id; + + /* + * Assign this source to a local APIC in a round-robin fashion. + */ + pic = isrc->is_pic; + apic_id = cpu_apic_ids[current_cpu]; + current_cpu++; + if (current_cpu >= num_cpus) + current_cpu = 0; + if (bootverbose || 1) { + printf("INTR: Assigning IRQ %d", pic->pic_vector(isrc)); + printf(" to local APIC %u\n", apic_id); + } + pic->pic_assign_cpu(isrc, apic_id); +} + +/* + * Add a local APIC ID to our list of valid local APIC IDs that can + * be destinations of interrupts. + */ +void +intr_add_cpu(u_int apic_id) +{ + + if (bootverbose || 1) + printf("INTR: Adding local APIC %d as a target\n", apic_id); + if (num_cpus >= MAXCPU) + panic("WARNING: Local APIC IDs exhausted!"); + cpu_apic_ids[num_cpus] = apic_id; + num_cpus++; +} + +/* + * Distribute all the interrupt sources among the available CPUs once the + * AP's have been launched. + */ +static void +intr_shuffle_irqs(void *arg __unused) +{ + struct intsrc *isrc; + int i; + + /* Don't bother on UP. */ + if (num_cpus <= 1) + return; + + /* Round-robin assign each enabled source a CPU. */ + mtx_lock_spin(&intr_table_lock); + assign_cpu = 1; + for (i = 0; i < NUM_IO_INTS; i++) { + isrc = interrupt_sources[i]; + if (isrc != NULL && isrc->is_enabled) + intr_assign_next_cpu(isrc); + } + mtx_unlock_spin(&intr_table_lock); +} +SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL) +#endif --- //depot/projects/smpng/sys/amd64/amd64/io_apic.c 2005/11/21 20:05:51 +++ //depot/user/jhb/intr/amd64/amd64/io_apic.c 2005/11/21 20:54:22 @@ -61,8 +61,6 @@ #define IRQ_SMI (NUM_IO_INTS + 3) #define IRQ_DISABLED (NUM_IO_INTS + 4) -#define DEST_NONE -1 - #define TODO printf("%s: not implemented!\n", __func__) static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures"); @@ -82,10 +80,10 @@ u_int io_irq; u_int io_intpin:8; u_int io_vector:8; + u_int io_cpu:8; u_int io_activehi:1; u_int io_edgetrigger:1; u_int io_masked:1; - int io_dest:5; int io_bus:4; }; @@ -114,7 +112,7 @@ enum intr_polarity pol); static void ioapic_suspend(struct intsrc *isrc); static void ioapic_resume(struct intsrc *isrc); -static void ioapic_program_destination(struct ioapic_intsrc *intpin); +static void ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id); static void ioapic_program_intpin(struct ioapic_intsrc *intpin); static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list); @@ -122,10 +120,10 @@ ioapic_eoi_source, ioapic_enable_intr, ioapic_vector, ioapic_source_pending, ioapic_suspend, ioapic_resume, - ioapic_config_intr }; - -static int bsp_id, current_cluster, logical_clusters, next_ioapic_base; -static u_int next_id, program_logical_dest; + ioapic_config_intr, ioapic_assign_cpu }; + +static int next_ioapic_base; +static u_int next_id; SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options"); static int enable_extint; @@ -273,14 +271,8 @@ } /* Set the destination. */ - if (intpin->io_dest == DEST_NONE) { - low = IOART_DESTPHY; - high = bsp_id << APIC_ID_SHIFT; - } else { - low = IOART_DESTLOG; - high = (intpin->io_dest << APIC_ID_CLUSTER_SHIFT | - APIC_ID_CLUSTER_ID) << APIC_ID_SHIFT; - } + low = IOART_DESTPHY; + high = intpin->io_cpu << APIC_ID_SHIFT; /* Program the rest of the low word. */ if (intpin->io_edgetrigger) @@ -312,7 +304,7 @@ default: KASSERT(intpin->io_vector != 0, ("No vector for IRQ %u", intpin->io_irq)); - low |= IOART_DELLOPRI | intpin->io_vector; + low |= IOART_DELFIXED | intpin->io_vector; } /* Write the values to the APIC. */ @@ -325,60 +317,31 @@ mtx_unlock_spin(&icu_lock); } -/* - * Program an individual intpin's logical destination. - */ static void -ioapic_program_destination(struct ioapic_intsrc *intpin) +ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id) { - struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + struct ioapic *io = (struct ioapic *)isrc->is_pic; - KASSERT(intpin->io_dest != DEST_NONE, - ("intpin not assigned to a cluster")); - if (bootverbose) { - printf("ioapic%u: routing intpin %u (", io->io_id, - intpin->io_intpin); + intpin->io_cpu = apic_id; + if (bootverbose || 1) { + printf("ioapic%u: Assigning ", io->io_id); ioapic_print_irq(intpin); - printf(") to cluster %u\n", intpin->io_dest); + printf(" to local APIC %u\n", intpin->io_cpu); } ioapic_program_intpin(intpin); } static void -ioapic_assign_cluster(struct ioapic_intsrc *intpin) -{ - - /* - * Assign this intpin to a logical APIC cluster in a - * round-robin fashion. We don't actually use the logical - * destination for this intpin until after all the CPU's - * have been started so that we don't end up with interrupts - * that don't go anywhere. Another alternative might be to - * start up the CPU's earlier so that they can handle interrupts - * sooner. - */ - intpin->io_dest = current_cluster; - current_cluster++; - if (current_cluster >= logical_clusters) - current_cluster = 0; - if (program_logical_dest) - ioapic_program_destination(intpin); -} - -static void ioapic_enable_intr(struct intsrc *isrc) { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; struct ioapic *io = (struct ioapic *)isrc->is_pic; - if (intpin->io_dest == DEST_NONE) { + if (intpin->io_vector == 0) { /* * Allocate an APIC vector for this interrupt pin. Once - * we have a vector we program the interrupt pin. Note - * that after we have booted ioapic_assign_cluster() - * will program the interrupt pin again, but it doesn't - * hurt to do that and trying to avoid that adds needless - * complication. + * we have a vector we program the interrupt pin. */ intpin->io_vector = apic_alloc_vector(intpin->io_irq); if (bootverbose) { @@ -388,7 +351,6 @@ printf(") to vector %u\n", intpin->io_vector); } ioapic_program_intpin(intpin); - ioapic_assign_cluster(intpin); apic_enable_vector(intpin->io_vector); } } @@ -469,22 +431,6 @@ } /* - * Allocate and return a logical cluster ID. Note that the first time - * this is called, it returns cluster 0. ioapic_enable_intr() treats - * the two cases of logical_clusters == 0 and logical_clusters == 1 the - * same: one cluster of ID 0 exists. The logical_clusters == 0 case is - * for UP kernels, which should never call this function. - */ -int -ioapic_next_logical_cluster(void) -{ - - if (logical_clusters >= APIC_MAX_CLUSTER) - panic("WARNING: Local APIC cluster IDs exhausted!"); - return (logical_clusters++); -} - -/* * Create a plain I/O APIC object. */ void * @@ -568,11 +514,10 @@ } /* - * Route interrupts to the BSP by default using physical - * addressing. Vectored interrupts get readdressed using - * logical IDs to CPU clusters when they are enabled. + * Route interrupts to the BSP by default. Interrupts may + * be routed to other CPUs later after they are enabled. */ - intpin->io_dest = DEST_NONE; + intpin->io_cpu = PCPU_GET(apic_id); if (bootverbose && intpin->io_irq != IRQ_DISABLED) { printf("ioapic%u: intpin %d -> ", io->io_id, i); ioapic_print_irq(intpin); @@ -778,29 +723,9 @@ printf("ioapic%u irqs %u-%u on motherboard\n", io->io_id, flags >> 4, flags & 0xf, io->io_intbase, io->io_intbase + io->io_numintr - 1); - bsp_id = PCPU_GET(apic_id); /* Register valid pins as interrupt sources. */ for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) if (pin->io_irq < NUM_IO_INTS) intr_register_source(&pin->io_intsrc); } - -/* - * Program all the intpins to use logical destinations once the AP's - * have been launched. - */ -static void -ioapic_set_logical_destinations(void *arg __unused) -{ - struct ioapic *io; - int i; - - program_logical_dest = 1; - STAILQ_FOREACH(io, &ioapic_list, io_next) - for (i = 0; i < io->io_numintr; i++) - if (io->io_pins[i].io_dest != DEST_NONE) - ioapic_program_destination(&io->io_pins[i]); -} -SYSINIT(ioapic_destinations, SI_SUB_SMP, SI_ORDER_SECOND, - ioapic_set_logical_destinations, NULL) --- //depot/projects/smpng/sys/amd64/amd64/local_apic.c 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/amd64/amd64/local_apic.c 2006/01/06 21:17:48 @@ -217,6 +217,7 @@ /* Set BSP's per-CPU local APIC ID. */ PCPU_SET(apic_id, lapic_id()); + intr_add_cpu(PCPU_GET(apic_id)); /* Local APIC timer interrupt. */ setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYSIGT, SEL_KPL, 0); @@ -279,7 +280,7 @@ lapic_setup(void) { struct lapic *la; - u_int32_t value, maxlvt; + u_int32_t maxlvt; register_t eflags; char buf[MAXCOMLEN + 1]; @@ -288,22 +289,14 @@ eflags = intr_disable(); maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + if (PCPU_GET(cpuid) != 0) { + snprintf(buf, sizeof(buf), "cpu%d: intrs", PCPU_GET(cpuid)); + intrcnt_add(buf, PCPU_PTR(intr_count)); + } + /* Initialize the TPR to allow all interrupts. */ lapic_set_tpr(0); - /* Use the cluster model for logical IDs. */ - value = lapic->dfr; - value &= ~APIC_DFR_MODEL_MASK; - value |= APIC_DFR_MODEL_CLUSTER; - lapic->dfr = value; - - /* Set this APIC's logical ID. */ - value = lapic->ldr; - value &= ~APIC_ID_MASK; - value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT | - 1 << la->la_cluster_id) << APIC_ID_SHIFT; - lapic->ldr = value; - /* Setup spurious vector and enable the local APIC. */ lapic_enable(); --- //depot/projects/smpng/sys/amd64/amd64/mp_machdep.c 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/amd64/amd64/mp_machdep.c 2006/01/06 21:17:48 @@ -160,7 +160,7 @@ static u_int boot_address; -static void set_logical_apic_ids(void); +static void set_interrupt_apic_ids(void); static int start_all_aps(void); static int start_ap(int apic_id); static void release_aps(void *dummy); @@ -405,7 +405,7 @@ hyperthreading_cpus = logical_cpus; } - set_logical_apic_ids(); + set_interrupt_apic_ids(); } @@ -596,33 +596,29 @@ */ /* - * Set the APIC logical IDs. - * - * We want to cluster logical CPU's within the same APIC ID cluster. - * Since logical CPU's are aligned simply filling in the clusters in - * APIC ID order works fine. Note that this does not try to balance - * the number of CPU's in each cluster. (XXX?) + * We tell the I/O APIC code about all the CPUs we want to receive + * interrupts. If we don't want certain CPUs to receive IRQs we + * can simply not tell the I/O APIC code about them in this function. + * We also do not tell it about the BSP since it tells itself about + * the BSP internally to work with UP kernels and on UP machines. */ static void -set_logical_apic_ids(void) +set_interrupt_apic_ids(void) { - u_int apic_id, cluster, cluster_id; + u_int apic_id; - /* Force us to allocate cluster 0 at the start. */ - cluster = -1; - cluster_id = APIC_MAX_INTRACLUSTER_ID; for (apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present) continue; - if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { - cluster = ioapic_next_logical_cluster(); - cluster_id = 0; - } else - cluster_id++; - if (bootverbose) - printf("APIC ID: physical %u, logical %u:%u\n", - apic_id, cluster, cluster_id); - lapic_set_logical_id(apic_id, cluster, cluster_id); + if (cpu_info[apic_id].cpu_bsp) + continue; +#if 0 + /* Don't let hyperthreads service interrupts. */ + if (hyperthreading_cpus > 1 && + apic_id % hyperthreading_cpus != 0) + continue; +#endif + intr_add_cpu(apic_id); } } --- //depot/projects/smpng/sys/amd64/include/apicvar.h 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/amd64/include/apicvar.h 2006/01/06 21:17:48 @@ -181,7 +181,6 @@ void *ioapic_create(uintptr_t addr, int32_t id, int intbase); int ioapic_disable_pin(void *cookie, u_int pin); int ioapic_get_vector(void *cookie, u_int pin); -int ioapic_next_logical_cluster(void); void ioapic_register(void *cookie); int ioapic_remap_vector(void *cookie, u_int pin, int vector); int ioapic_set_bus(void *cookie, u_int pin, int bus_type); --- //depot/projects/smpng/sys/amd64/include/intr_machdep.h 2005/12/14 21:21:48 +++ //depot/user/jhb/intr/amd64/include/intr_machdep.h 2005/12/21 21:37:11 @@ -85,6 +85,7 @@ void (*pic_resume)(struct intsrc *); int (*pic_config_intr)(struct intsrc *, enum intr_trigger, enum intr_polarity); + void (*pic_assign_cpu)(struct intsrc *, u_int apic_id); }; /* Flags for pic_disable_source() */ @@ -105,6 +106,7 @@ u_long *is_count; u_long *is_straycount; u_int is_index; + u_int is_enabled:1; }; struct trapframe; @@ -117,6 +119,11 @@ enum intr_trigger elcr_read_trigger(u_int irq); void elcr_resume(void); void elcr_write_trigger(u_int irq, enum intr_trigger trigger); +#ifdef SMP +void intr_add_cpu(u_int apic_id); +#else +#define intr_add_cpu(apic_id) +#endif int intr_add_handler(const char *name, int vector, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); int intr_config_intr(int vector, enum intr_trigger trig, --- //depot/projects/smpng/sys/amd64/include/pcpu.h 2005/03/24 19:46:32 +++ //depot/user/jhb/intr/amd64/include/pcpu.h 2005/11/18 15:33:18 @@ -48,6 +48,7 @@ struct amd64tss *pc_tssp; \ register_t pc_rsp0; \ register_t pc_scratch_rsp; /* User %rsp in syscall */ \ + u_long *pc_intr_count; \ u_int pc_apic_id; \ u_int pc_acpi_id /* ACPI CPU id */ --- //depot/projects/smpng/sys/amd64/isa/atpic.c 2005/12/14 21:21:48 +++ //depot/user/jhb/intr/amd64/isa/atpic.c 2005/12/21 21:37:11 @@ -108,8 +108,8 @@ #define ATPIC(io, base, eoi, imenptr) \ { { atpic_enable_source, atpic_disable_source, (eoi), \ atpic_enable_intr, atpic_vector, atpic_source_pending, NULL, \ - atpic_resume, atpic_config_intr }, (io), (base), \ - IDT_IO_INTS + (base), (imenptr) } + atpic_resume, atpic_config_intr, atpic_assign_cpu }, (io), \ + (base), IDT_IO_INTS + (base), (imenptr) } #define INTSRC(irq) \ { { &atpics[(irq) / 8].at_pic }, IDTVEC(atpic_intr ## irq ), \ @@ -142,6 +142,7 @@ static int atpic_source_pending(struct intsrc *isrc); static int atpic_config_intr(struct intsrc *isrc, enum intr_trigger trig, enum intr_polarity pol); +static void atpic_assign_cpu(struct intsrc *isrc, u_int apic_id); static void i8259_init(struct atpic *pic, int slave); static struct atpic atpics[] = { @@ -353,6 +354,17 @@ } static void +atpic_assign_cpu(struct intsrc *isrc, u_int apic_id) +{ + + /* + * 8259A's are only used in UP in which case all interrupts always + * go to the sole CPU and this function shouldn't even be called. + */ + panic("%s: bad cookie", __func__); +} + +static void i8259_init(struct atpic *pic, int slave) { int imr_addr; --- //depot/projects/smpng/sys/i386/i386/intr_machdep.c 2005/12/07 22:12:14 +++ //depot/user/jhb/intr/i386/i386/intr_machdep.c 2005/12/08 18:03:18 @@ -63,6 +63,12 @@ static struct intsrc *interrupt_sources[NUM_IO_INTS]; static struct mtx intr_table_lock; +#ifdef SMP +static int assign_cpu; + +static void intr_assign_next_cpu(struct intsrc *isrc); +#endif + static void intr_init(void *__dummy); static void intrcnt_setname(const char *name, int index); static void intrcnt_updatename(struct intsrc *is); @@ -93,6 +99,7 @@ } intrcnt_register(isrc); interrupt_sources[vector] = isrc; + isrc->is_enabled = 0; mtx_unlock_spin(&intr_table_lock); return (0); } @@ -118,7 +125,17 @@ intr_priority(flags), flags, cookiep); if (error == 0) { intrcnt_updatename(isrc); - isrc->is_pic->pic_enable_intr(isrc); + mtx_lock_spin(&intr_table_lock); + if (!isrc->is_enabled) { + isrc->is_enabled = 1; +#ifdef SMP + if (assign_cpu) + intr_assign_next_cpu(isrc); +#endif + mtx_unlock_spin(&intr_table_lock); + isrc->is_pic->pic_enable_intr(isrc); + } else + mtx_unlock_spin(&intr_table_lock); isrc->is_pic->pic_enable_source(isrc); } return (error); @@ -166,6 +183,7 @@ */ (*isrc->is_count)++; PCPU_LAZY_INC(cnt.v_intr); + (*PCPU_GET(intr_count))++; ie = isrc->is_event; @@ -311,6 +329,7 @@ intrcnt_setname("???", 0); intrcnt_index = 1; mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); + intrcnt_add("cpu0: intrs", PCPU_PTR(intr_count)); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -335,3 +354,76 @@ db_dump_intr_event((*isrc)->is_event, verbose); } #endif + +#ifdef SMP +/* + * Support for balancing interrupt sources across CPUs. For now we just + * allocate CPUs round-robin. + */ + +static u_int cpu_apic_ids[MAXCPU]; +static int current_cpu, num_cpus; + +static void +intr_assign_next_cpu(struct intsrc *isrc) +{ + struct pic *pic; + u_int apic_id; + + /* + * Assign this source to a local APIC in a round-robin fashion. + */ + pic = isrc->is_pic; + apic_id = cpu_apic_ids[current_cpu]; + current_cpu++; + if (current_cpu >= num_cpus) + current_cpu = 0; + if (bootverbose || 1) { + printf("INTR: Assigning IRQ %d", pic->pic_vector(isrc)); + printf(" to local APIC %u\n", apic_id); + } + pic->pic_assign_cpu(isrc, apic_id); +} + +/* + * Add a local APIC ID to our list of valid local APIC IDs that can + * be destinations of interrupts. + */ +void +intr_add_cpu(u_int apic_id) +{ + + if (bootverbose || 1) + printf("INTR: Adding local APIC %d as a target\n", apic_id); + if (num_cpus >= MAXCPU) + panic("WARNING: Local APIC IDs exhausted!"); + cpu_apic_ids[num_cpus] = apic_id; + num_cpus++; +} + +/* + * Distribute all the interrupt sources among the available CPUs once the + * AP's have been launched. + */ +static void +intr_shuffle_irqs(void *arg __unused) +{ + struct intsrc *isrc; + int i; + + /* Don't bother on UP. */ + if (num_cpus <= 1) + return; + + /* Round-robin assign each enabled source a CPU. */ + mtx_lock_spin(&intr_table_lock); + assign_cpu = 1; + for (i = 0; i < NUM_IO_INTS; i++) { + isrc = interrupt_sources[i]; + if (isrc != NULL && isrc->is_enabled) + intr_assign_next_cpu(isrc); + } + mtx_unlock_spin(&intr_table_lock); +} +SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL) +#endif --- //depot/projects/smpng/sys/i386/i386/io_apic.c 2005/11/21 20:05:51 +++ //depot/user/jhb/intr/i386/i386/io_apic.c 2005/11/21 20:54:22 @@ -60,8 +60,6 @@ #define IRQ_SMI (NUM_IO_INTS + 3) #define IRQ_DISABLED (NUM_IO_INTS + 4) -#define DEST_NONE -1 - #define TODO printf("%s: not implemented!\n", __func__) static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures"); @@ -85,10 +83,10 @@ u_int io_irq; u_int io_intpin:8; u_int io_vector:8; + u_int io_cpu:8; u_int io_activehi:1; u_int io_edgetrigger:1; u_int io_masked:1; - int io_dest:5; int io_bus:4; }; @@ -117,7 +115,7 @@ enum intr_polarity pol); static void ioapic_suspend(struct intsrc *isrc); static void ioapic_resume(struct intsrc *isrc); -static void ioapic_program_destination(struct ioapic_intsrc *intpin); +static void ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id); static void ioapic_program_intpin(struct ioapic_intsrc *intpin); static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list); @@ -125,10 +123,10 @@ ioapic_eoi_source, ioapic_enable_intr, ioapic_vector, ioapic_source_pending, ioapic_suspend, ioapic_resume, - ioapic_config_intr }; - -static int bsp_id, current_cluster, logical_clusters, next_ioapic_base; -static u_int next_id, program_logical_dest; + ioapic_config_intr, ioapic_assign_cpu }; + +static int next_ioapic_base; +static u_int next_id; SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options"); static int enable_extint; @@ -276,14 +274,8 @@ } /* Set the destination. */ - if (intpin->io_dest == DEST_NONE) { - low = IOART_DESTPHY; - high = bsp_id << APIC_ID_SHIFT; - } else { - low = IOART_DESTLOG; - high = (intpin->io_dest << APIC_ID_CLUSTER_SHIFT | - APIC_ID_CLUSTER_ID) << APIC_ID_SHIFT; - } + low = IOART_DESTPHY; + high = intpin->io_cpu << APIC_ID_SHIFT; /* Program the rest of the low word. */ if (intpin->io_edgetrigger) @@ -315,7 +307,7 @@ default: KASSERT(intpin->io_vector != 0, ("No vector for IRQ %u", intpin->io_irq)); - low |= IOART_DELLOPRI | intpin->io_vector; + low |= IOART_DELFIXED | intpin->io_vector; } /* Write the values to the APIC. */ @@ -328,60 +320,31 @@ mtx_unlock_spin(&icu_lock); } -/* - * Program an individual intpin's logical destination. - */ static void -ioapic_program_destination(struct ioapic_intsrc *intpin) +ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id) { - struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + struct ioapic *io = (struct ioapic *)isrc->is_pic; - KASSERT(intpin->io_dest != DEST_NONE, - ("intpin not assigned to a cluster")); - if (bootverbose) { - printf("ioapic%u: routing intpin %u (", io->io_id, - intpin->io_intpin); + intpin->io_cpu = apic_id; + if (bootverbose || 1) { + printf("ioapic%u: Assigning ", io->io_id); ioapic_print_irq(intpin); - printf(") to cluster %u\n", intpin->io_dest); + printf(" to local APIC %u\n", intpin->io_cpu); } ioapic_program_intpin(intpin); } static void -ioapic_assign_cluster(struct ioapic_intsrc *intpin) -{ - - /* - * Assign this intpin to a logical APIC cluster in a - * round-robin fashion. We don't actually use the logical - * destination for this intpin until after all the CPU's - * have been started so that we don't end up with interrupts - * that don't go anywhere. Another alternative might be to - * start up the CPU's earlier so that they can handle interrupts - * sooner. - */ - intpin->io_dest = current_cluster; - current_cluster++; - if (current_cluster >= logical_clusters) - current_cluster = 0; - if (program_logical_dest) - ioapic_program_destination(intpin); -} - -static void ioapic_enable_intr(struct intsrc *isrc) { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; struct ioapic *io = (struct ioapic *)isrc->is_pic; - if (intpin->io_dest == DEST_NONE) { + if (intpin->io_vector == 0) { /* * Allocate an APIC vector for this interrupt pin. Once - * we have a vector we program the interrupt pin. Note - * that after we have booted ioapic_assign_cluster() - * will program the interrupt pin again, but it doesn't - * hurt to do that and trying to avoid that adds needless - * complication. + * we have a vector we program the interrupt pin. */ intpin->io_vector = apic_alloc_vector(intpin->io_irq); if (bootverbose) { @@ -391,7 +354,6 @@ printf(") to vector %u\n", intpin->io_vector); } ioapic_program_intpin(intpin); - ioapic_assign_cluster(intpin); apic_enable_vector(intpin->io_vector); } } @@ -472,22 +434,6 @@ } /* - * Allocate and return a logical cluster ID. Note that the first time - * this is called, it returns cluster 0. ioapic_enable_intr() treats - * the two cases of logical_clusters == 0 and logical_clusters == 1 the - * same: one cluster of ID 0 exists. The logical_clusters == 0 case is - * for UP kernels, which should never call this function. - */ -int -ioapic_next_logical_cluster(void) -{ - - if (logical_clusters >= APIC_MAX_CLUSTER) - panic("WARNING: Local APIC cluster IDs exhausted!"); - return (logical_clusters++); -} - -/* * Create a plain I/O APIC object. */ void * @@ -571,11 +517,10 @@ } /* - * Route interrupts to the BSP by default using physical - * addressing. Vectored interrupts get readdressed using - * logical IDs to CPU clusters when they are enabled. + * Route interrupts to the BSP by default. Interrupts may + * be routed to other CPUs later after they are enabled. */ - intpin->io_dest = DEST_NONE; + intpin->io_cpu = PCPU_GET(apic_id); if (bootverbose && intpin->io_irq != IRQ_DISABLED) { printf("ioapic%u: intpin %d -> ", io->io_id, i); ioapic_print_irq(intpin); @@ -781,29 +726,9 @@ printf("ioapic%u irqs %u-%u on motherboard\n", io->io_id, flags >> 4, flags & 0xf, io->io_intbase, io->io_intbase + io->io_numintr - 1); - bsp_id = PCPU_GET(apic_id); /* Register valid pins as interrupt sources. */ for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) if (pin->io_irq < NUM_IO_INTS) intr_register_source(&pin->io_intsrc); } - -/* - * Program all the intpins to use logical destinations once the AP's - * have been launched. - */ -static void -ioapic_set_logical_destinations(void *arg __unused) -{ - struct ioapic *io; - int i; - - program_logical_dest = 1; - STAILQ_FOREACH(io, &ioapic_list, io_next) - for (i = 0; i < io->io_numintr; i++) - if (io->io_pins[i].io_dest != DEST_NONE) - ioapic_program_destination(&io->io_pins[i]); -} -SYSINIT(ioapic_destinations, SI_SUB_SMP, SI_ORDER_SECOND, - ioapic_set_logical_destinations, NULL) --- //depot/projects/smpng/sys/i386/i386/local_apic.c 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/i386/i386/local_apic.c 2006/01/06 21:17:48 @@ -218,6 +218,7 @@ /* Set BSP's per-CPU local APIC ID. */ PCPU_SET(apic_id, lapic_id()); + intr_add_cpu(PCPU_GET(apic_id)); /* Local APIC timer interrupt. */ setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYS386IGT, SEL_KPL, @@ -281,7 +282,7 @@ lapic_setup(void) { struct lapic *la; - u_int32_t value, maxlvt; + u_int32_t maxlvt; register_t eflags; char buf[MAXCOMLEN + 1]; @@ -290,22 +291,14 @@ eflags = intr_disable(); maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + if (PCPU_GET(cpuid) != 0) { + snprintf(buf, sizeof(buf), "cpu%d: intrs", PCPU_GET(cpuid)); + intrcnt_add(buf, PCPU_PTR(intr_count)); + } + /* Initialize the TPR to allow all interrupts. */ lapic_set_tpr(0); - /* Use the cluster model for logical IDs. */ - value = lapic->dfr; - value &= ~APIC_DFR_MODEL_MASK; - value |= APIC_DFR_MODEL_CLUSTER; - lapic->dfr = value; - - /* Set this APIC's logical ID. */ - value = lapic->ldr; - value &= ~APIC_ID_MASK; - value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT | - 1 << la->la_cluster_id) << APIC_ID_SHIFT; - lapic->ldr = value; - /* Setup spurious vector and enable the local APIC. */ lapic_enable(); --- //depot/projects/smpng/sys/i386/i386/mp_machdep.c 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/i386/i386/mp_machdep.c 2006/01/06 21:17:48 @@ -217,7 +217,7 @@ static u_int boot_address; -static void set_logical_apic_ids(void); +static void set_interrupt_apic_ids(void); static int start_all_aps(void); static void install_ap_tramp(void); static int start_ap(int apic_id); @@ -454,7 +454,7 @@ hyperthreading_cpus = logical_cpus; } - set_logical_apic_ids(); + set_interrupt_apic_ids(); } @@ -645,33 +645,29 @@ */ /* - * Set the APIC logical IDs. - * - * We want to cluster logical CPU's within the same APIC ID cluster. - * Since logical CPU's are aligned simply filling in the clusters in - * APIC ID order works fine. Note that this does not try to balance - * the number of CPU's in each cluster. (XXX?) + * We tell the I/O APIC code about all the CPUs we want to receive + * interrupts. If we don't want certain CPUs to receive IRQs we + * can simply not tell the I/O APIC code about them in this function. + * We also do not tell it about the BSP since it tells itself about + * the BSP internally to work with UP kernels and on UP machines. */ static void -set_logical_apic_ids(void) +set_interrupt_apic_ids(void) { - u_int apic_id, cluster, cluster_id; + u_int apic_id; - /* Force us to allocate cluster 0 at the start. */ - cluster = -1; - cluster_id = APIC_MAX_INTRACLUSTER_ID; for (apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present) continue; - if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { - cluster = ioapic_next_logical_cluster(); - cluster_id = 0; - } else - cluster_id++; - if (bootverbose) - printf("APIC ID: physical %u, logical %u:%u\n", - apic_id, cluster, cluster_id); - lapic_set_logical_id(apic_id, cluster, cluster_id); + if (cpu_info[apic_id].cpu_bsp) + continue; +#if 0 + /* Don't let hyperthreads service interrupts. */ + if (hyperthreading_cpus > 1 && + apic_id % hyperthreading_cpus != 0) + continue; +#endif + intr_add_cpu(apic_id); } } --- //depot/projects/smpng/sys/i386/include/apicvar.h 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/i386/include/apicvar.h 2006/01/06 21:17:48 @@ -203,7 +203,6 @@ void *ioapic_create(uintptr_t addr, int32_t id, int intbase); int ioapic_disable_pin(void *cookie, u_int pin); int ioapic_get_vector(void *cookie, u_int pin); -int ioapic_next_logical_cluster(void); void ioapic_register(void *cookie); int ioapic_remap_vector(void *cookie, u_int pin, int vector); int ioapic_set_bus(void *cookie, u_int pin, int bus_type); --- //depot/projects/smpng/sys/i386/include/intr_machdep.h 2005/12/07 22:12:14 +++ //depot/user/jhb/intr/i386/include/intr_machdep.h 2005/12/08 18:03:18 @@ -85,6 +85,7 @@ void (*pic_resume)(struct intsrc *); int (*pic_config_intr)(struct intsrc *, enum intr_trigger, enum intr_polarity); + void (*pic_assign_cpu)(struct intsrc *, u_int apic_id); }; /* Flags for pic_disable_source() */ @@ -105,6 +106,7 @@ u_long *is_count; u_long *is_straycount; u_int is_index; + u_int is_enabled:1; }; struct trapframe; @@ -117,6 +119,11 @@ enum intr_trigger elcr_read_trigger(u_int irq); void elcr_resume(void); void elcr_write_trigger(u_int irq, enum intr_trigger trigger); +#ifdef SMP +void intr_add_cpu(u_int apic_id); +#else +#define intr_add_cpu(apic_id) +#endif int intr_add_handler(const char *name, int vector, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); int intr_config_intr(int vector, enum intr_trigger trig, --- //depot/projects/smpng/sys/i386/include/pcpu.h 2006/01/06 20:21:45 +++ //depot/user/jhb/intr/i386/include/pcpu.h 2006/01/06 21:17:48 @@ -53,6 +53,7 @@ struct segment_descriptor *pc_tss_gdt; \ struct segment_descriptor *pc_fsgs_gdt; \ int pc_currentldt; \ + u_long *pc_intr_count; \ u_int pc_acpi_id; \ u_int pc_apic_id; \ int pc_private_tss /* flag indicating private tss */ --- //depot/projects/smpng/sys/i386/isa/atpic.c 2005/12/14 21:21:48 +++ //depot/user/jhb/intr/i386/isa/atpic.c 2005/12/21 21:37:11 @@ -126,8 +126,8 @@ #define ATPIC(io, base, eoi, imenptr) \ { { atpic_enable_source, atpic_disable_source, (eoi), \ atpic_enable_intr, atpic_vector, atpic_source_pending, NULL, \ - atpic_resume, atpic_config_intr }, (io), (base), \ - IDT_IO_INTS + (base), (imenptr) } + atpic_resume, atpic_config_intr, atpic_assign_cpu }, (io), \ + (base), IDT_IO_INTS + (base), (imenptr) } #define INTSRC(irq) \ { { &atpics[(irq) / 8].at_pic }, IDTVEC(atpic_intr ## irq ), \ @@ -160,6 +160,7 @@ static int atpic_source_pending(struct intsrc *isrc); static int atpic_config_intr(struct intsrc *isrc, enum intr_trigger trig, enum intr_polarity pol); +static void atpic_assign_cpu(struct intsrc *isrc, u_int apic_id); static void i8259_init(struct atpic *pic, int slave); static struct atpic atpics[] = { @@ -385,6 +386,17 @@ } static void +atpic_assign_cpu(struct intsrc *isrc, u_int apic_id) +{ + + /* + * 8259A's are only used in UP in which case all interrupts always + * go to the sole CPU and this function shouldn't even be called. + */ + panic("%s: bad cookie", __func__); +} + +static void i8259_init(struct atpic *pic, int slave) { int imr_addr;