--- //depot/projects/smpng/sys/amd64/amd64/intr_machdep.c 2007/03/07 19:17:11 +++ //depot/user/jhb/intr/amd64/amd64/intr_machdep.c 2007/03/21 18:34:47 @@ -80,6 +80,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -135,7 +136,8 @@ if (interrupt_sources[vector] != NULL) return (EEXIST); error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", + vector); if (error) return (error); mtx_lock_spin(&intr_table_lock); @@ -307,12 +310,10 @@ #ifndef DEV_ATPIC atpic_reset(); #endif - mtx_lock_spin(&intr_table_lock); STAILQ_FOREACH(pic, &pics, pics) { if (pic->pic_resume != NULL) pic->pic_resume(pic); } - mtx_unlock_spin(&intr_table_lock); } void @@ -320,12 +321,32 @@ { struct pic *pic; - mtx_lock_spin(&intr_table_lock); STAILQ_FOREACH(pic, &pics, pics) { if (pic->pic_suspend != NULL) pic->pic_suspend(pic); } - mtx_unlock_spin(&intr_table_lock); +} + +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu) { + isrc = arg; + mtx_lock_spin(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + mtx_unlock_spin(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif } static void @@ -378,7 +399,12 @@ intrcnt_setname("???", 0); intrcnt_index = 1; STAILQ_INIT(&pics); +#ifdef BIND_ALL + mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN | MTX_RECURSE); +#else mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); +#endif + intrcnt_add("cpu0: intrs", PCPU_PTR(intr_count)); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -437,15 +463,15 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); +#ifdef BIND_ALL + intr_event_bind(isrc->is_event, current_cpu); +#else + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); +#endif do { current_cpu++; if (current_cpu >= num_cpus) @@ -453,6 +479,18 @@ } while (!(intr_cpus & (1 << current_cpu))); } +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -482,7 +520,7 @@ int i; /* Don't bother on UP. */ - if (num_cpus <= 1) + if (mp_ncpus <= 1) return; /* Round-robin assign a CPU to each enabled source. */ @@ -490,8 +528,18 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_enabled) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_enabled) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + } } mtx_unlock_spin(&intr_table_lock); } --- //depot/projects/smpng/sys/amd64/amd64/sys_machdep.c 2005/07/13 18:25:40 +++ //depot/user/jhb/intr/amd64/amd64/sys_machdep.c 2006/04/20 17:20:36 @@ -35,9 +35,12 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -62,6 +65,9 @@ struct pcb *pcb = curthread->td_pcb; uint32_t i386base; uint64_t a64base; +#ifdef SMP + struct amd64_intr_bind_args bargs; +#endif switch(uap->op) { case I386_GET_FSBASE: @@ -90,6 +96,15 @@ critical_exit(); } break; +#ifdef SMP + /* ABI and API compatible with I386_INTR_BIND. */ + case AMD64_INTR_BIND: + error = copyin(uap->parms, &bargs, + sizeof(struct amd64_intr_bind_args)); + if (error == 0) + error = intr_bind(bargs.vector, bargs.cpu); + break; +#endif case AMD64_GET_FSBASE: error = copyout(&pcb->pcb_fsbase, uap->parms, sizeof(pcb->pcb_fsbase)); break; --- //depot/projects/smpng/sys/amd64/include/intr_machdep.h 2007/03/07 19:17:11 +++ //depot/user/jhb/intr/amd64/include/intr_machdep.h 2007/03/07 19:43:15 @@ -136,6 +136,9 @@ int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); +#ifdef SMP +int intr_bind(u_int vector, u_int cpu); +#endif int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); --- //depot/projects/smpng/sys/amd64/include/sysarch.h 2005/04/14 18:55:16 +++ //depot/user/jhb/intr/amd64/include/sysarch.h 2006/04/20 17:20:36 @@ -39,6 +39,7 @@ #define I386_SET_FSBASE 8 #define I386_GET_GSBASE 9 #define I386_SET_GSBASE 10 +#define AMD64_INTR_BIND 11 /* Leave space for 0-127 for to avoid translating syscalls */ #define AMD64_GET_FSBASE 128 @@ -46,6 +47,11 @@ #define AMD64_GET_GSBASE 130 #define AMD64_SET_GSBASE 131 +struct amd64_intr_bind_args { + unsigned int vector; + unsigned int cpu; +}; + #ifndef _KERNEL #include @@ -54,6 +60,7 @@ int amd64_get_gsbase(void **); int amd64_set_fsbase(void *); int amd64_set_gsbase(void *); +int amd64_intr_bind(unsigned int, unsigned int); int sysarch(int, void *); __END_DECLS #endif --- //depot/projects/smpng/sys/arm/arm/intr.c 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/arm/arm/intr.c 2007/03/21 18:31:28 @@ -69,7 +69,7 @@ event = intr_events[irq]; if (event == NULL) { error = intr_event_create(&event, (void *)irq, 0, - (void (*)(void *))arm_unmask_irq, "intr%d:", irq); + (void (*)(void *))arm_unmask_irq, NULL, "intr%d:", irq); if (error) return; intr_events[irq] = event; --- //depot/projects/smpng/sys/i386/i386/intr_machdep.c 2007/03/07 19:17:11 +++ //depot/user/jhb/intr/i386/i386/intr_machdep.c 2007/03/21 18:34:47 @@ -71,6 +71,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -126,7 +127,8 @@ if (interrupt_sources[vector] != NULL) return (EEXIST); error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", + vector); if (error) return (error); mtx_lock_spin(&intr_table_lock); @@ -295,12 +298,10 @@ { struct pic *pic; - mtx_lock_spin(&intr_table_lock); STAILQ_FOREACH(pic, &pics, pics) { if (pic->pic_resume != NULL) pic->pic_resume(pic); } - mtx_unlock_spin(&intr_table_lock); } void @@ -308,12 +309,32 @@ { struct pic *pic; - mtx_lock_spin(&intr_table_lock); STAILQ_FOREACH(pic, &pics, pics) { if (pic->pic_suspend != NULL) pic->pic_suspend(pic); } - mtx_unlock_spin(&intr_table_lock); +} + +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu) { + isrc = arg; + mtx_lock_spin(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + mtx_unlock_spin(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif } static void @@ -366,7 +387,12 @@ intrcnt_setname("???", 0); intrcnt_index = 1; STAILQ_INIT(&pics); +#ifdef BIND_ALL + mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN | MTX_RECURSE); +#else mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); +#endif + intrcnt_add("cpu0: intrs", PCPU_PTR(intr_count)); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -403,15 +429,15 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); +#ifdef BIND_ALL + intr_event_bind(isrc->is_event, current_cpu); +#else + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); +#endif do { current_cpu++; if (current_cpu >= num_cpus) @@ -419,6 +445,18 @@ } while (!(intr_cpus & (1 << current_cpu))); } +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -448,7 +486,7 @@ int i; /* Don't bother on UP. */ - if (num_cpus <= 1) + if (mp_ncpus <= 1) return; /* Round-robin assign a CPU to each enabled source. */ @@ -456,8 +494,18 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_enabled) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_enabled) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + } } mtx_unlock_spin(&intr_table_lock); } --- //depot/projects/smpng/sys/i386/i386/sys_machdep.c 2006/11/08 19:13:38 +++ //depot/user/jhb/intr/i386/i386/sys_machdep.c 2006/11/08 20:24:00 @@ -37,6 +37,8 @@ #include #include +#include +#include #include #include #include @@ -51,6 +53,7 @@ #include #include +#include #include #include #include @@ -92,6 +95,9 @@ union { struct i386_ldt_args largs; struct i386_ioperm_args iargs; +#ifdef SMP + struct i386_intr_bind_args bargs; +#endif } kargs; uint32_t base; struct segment_descriptor sd, *sdp; @@ -212,6 +218,14 @@ load_gs(GSEL(GUGS_SEL, SEL_UPL)); } break; +#ifdef SMP + case I386_INTR_BIND: + error = copyin(uap->parms, &kargs.bargs, + sizeof(struct i386_intr_bind_args)); + if (error == 0) + error = intr_bind(kargs.bargs.vector, kargs.bargs.cpu); + break; +#endif default: error = EINVAL; break; --- //depot/projects/smpng/sys/i386/include/intr_machdep.h 2007/03/07 19:17:11 +++ //depot/user/jhb/intr/i386/include/intr_machdep.h 2007/03/07 19:43:15 @@ -132,6 +132,9 @@ #endif int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); +#ifdef SMP +int intr_bind(u_int vector, u_int cpu); +#endif int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); --- //depot/projects/smpng/sys/i386/include/sysarch.h 2005/04/14 18:55:16 +++ //depot/user/jhb/intr/i386/include/sysarch.h 2006/03/24 22:02:35 @@ -47,6 +47,7 @@ #define I386_SET_FSBASE 8 #define I386_GET_GSBASE 9 #define I386_SET_GSBASE 10 +#define I386_INTR_BIND 11 /* These four only exist when running an i386 binary on amd64 */ #define _AMD64_GET_FSBASE 128 @@ -71,6 +72,11 @@ char *sub_args; /* args */ }; +struct i386_intr_bind_args { + unsigned int vector; + unsigned int cpu; +}; + #ifndef _KERNEL #include @@ -94,6 +100,7 @@ int i386_set_gsbase(void *); int i386_set_watch(int, unsigned int, int, int, struct dbreg *); int i386_clr_watch(int, struct dbreg *); +int i386_intr_bind(unsigned int, unsigned int); int sysarch(int, void *); __END_DECLS #else --- //depot/projects/smpng/sys/ia64/ia64/interrupt.c 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/ia64/ia64/interrupt.c 2007/03/21 18:31:28 @@ -320,7 +320,7 @@ bcopy(name, intrname, strlen(name)); } errcode = intr_event_create(&i->event, (void *)vector, 0, - (void (*)(void *))ia64_send_eoi, "intr:"); + (void (*)(void *))ia64_send_eoi, NULL, "intr:"); if (errcode) { free(i, M_DEVBUF); return errcode; --- //depot/projects/smpng/sys/kern/kern_intr.c 2007/03/07 19:17:11 +++ //depot/user/jhb/intr/kern/kern_intr.c 2007/03/21 18:32:48 @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -229,7 +230,8 @@ int intr_event_create(struct intr_event **event, void *source, int flags, - void (*enable)(void *), const char *fmt, ...) + void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, + ...) { struct intr_event *ie; va_list ap; @@ -240,7 +242,9 @@ ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_enable = enable; + ie->ie_assign_cpu = assign_cpu; ie->ie_flags = flags; + ie->ie_cpu = NOCPU; TAILQ_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -257,6 +261,31 @@ return (0); } +/* + * Bind an interrupt event to the specified CPU. Note that this just binds + * any associated ithreads to the specified CPU and doesn't currently call + * the MD code to bind the primary interrupt context to the specified CPU. + * Using a cpu id of NOCPU unbinds the interrupt event. + */ +int +intr_event_bind(struct intr_event *ie, u_char cpu) +{ + int error; + + /* Need a CPU to bind to. */ + if (cpu != NOCPU && CPU_ABSENT(cpu)) + return (EINVAL); + + if (ie->ie_assign_cpu == NULL) + return (EOPNOTSUPP); + + error = ie->ie_assign_cpu(ie, cpu); + mtx_lock_spin(&sched_lock); + ie->ie_cpu = cpu; + mtx_unlock_spin(&sched_lock); + return (0); +} + int intr_event_destroy(struct intr_event *ie) { @@ -581,7 +610,7 @@ if (!(ie->ie_flags & IE_SOFT)) return (EINVAL); } else { - error = intr_event_create(&ie, NULL, IE_SOFT, NULL, + error = intr_event_create(&ie, NULL, IE_SOFT, NULL, NULL, "swi%d:", pri); if (error) return (error); @@ -726,6 +755,7 @@ struct intr_event *ie; struct thread *td; struct proc *p; + u_char cpu; td = curthread; p = td->td_proc; @@ -734,6 +764,7 @@ ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; + cpu = NOCPU; /* * As long as we have interrupts outstanding, go through the @@ -779,6 +810,21 @@ ie->ie_count = 0; mi_switch(SW_VOL, NULL); } + +#ifdef SMP + /* + * Ensure we are bound to the correct CPU. We can't + * move ithreads until SMP is running however, so just + * leave interrupts on the boor CPU during boot. + */ + if (ie->ie_cpu != cpu && smp_started) { + cpu = ie->ie_cpu; + if (cpu == NOCPU) + sched_unbind(td); + else + sched_bind(td, cpu); + } +#endif mtx_unlock_spin(&sched_lock); } } @@ -882,6 +928,8 @@ db_printf("(pid %d)", it->it_thread->td_proc->p_pid); else db_printf("(no thread)"); + if (ie->ie_cpu != NOCPU) + db_printf(" (CPU %d)", ie->ie_cpu); if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || (it != NULL && it->it_need)) { db_printf(" {"); --- //depot/projects/smpng/sys/powerpc/powerpc/intr_machdep.c 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/powerpc/powerpc/intr_machdep.c 2007/03/21 18:31:28 @@ -153,7 +153,7 @@ if (i == NULL) return (ENOMEM); error = intr_event_create(&i->event, (void *)irq, 0, - (void (*)(void *))irq_enable, "irq%d:", irq); + (void (*)(void *))irq_enable, NULL, "irq%d:", irq); if (error) { free(i, M_INTR); return (error); --- //depot/projects/smpng/sys/sparc64/sparc64/intr_machdep.c 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/sparc64/sparc64/intr_machdep.c 2007/03/21 18:31:28 @@ -288,7 +288,7 @@ mtx_unlock_spin(&intr_table_lock); if (ie == NULL) { errcode = intr_event_create(&ie, (void *)(intptr_t)vec, 0, NULL, - "vec%d:", vec); + NULL, "vec%d:", vec); if (errcode) return (errcode); mtx_lock_spin(&intr_table_lock); --- //depot/projects/smpng/sys/sun4v/sun4v/intr_machdep.c 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/sun4v/sun4v/intr_machdep.c 2007/03/21 18:31:28 @@ -348,7 +348,7 @@ mtx_unlock_spin(&intr_table_lock); if (ie == NULL) { errcode = intr_event_create(&ie, (void *)(intptr_t)vec, 0, NULL, - "vec%d:", vec); + NULL, "vec%d:", vec); if (errcode) return (errcode); mtx_lock_spin(&intr_table_lock); --- //depot/projects/smpng/sys/sys/interrupt.h 2007/02/23 17:53:09 +++ //depot/user/jhb/intr/sys/interrupt.h 2007/03/21 18:31:28 @@ -72,9 +76,11 @@ void *ie_source; /* Cookie used by MD code. */ struct intr_thread *ie_thread; /* Thread we are connected to. */ void (*ie_enable)(void *); + int (*ie_assign_cpu)(void *, u_char); int ie_flags; int ie_count; /* Loop counter. */ int ie_warned; /* Warned about interrupt storm. */ + u_char ie_cpu; /* CPU this event is bound to. */ }; /* Interrupt event flags kept in ie_flags. */ @@ -117,9 +123,11 @@ int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep); +int intr_event_bind(struct intr_event *ie, u_char cpu); int intr_event_create(struct intr_event **event, void *source, - int flags, void (*enable)(void *), const char *fmt, ...) - __printflike(5, 6); + int flags, void (*enable)(void *), + int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(6, 7); int intr_event_destroy(struct intr_event *ie); int intr_event_remove_handler(void *cookie); int intr_event_schedule_thread(struct intr_event *ie);