diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c index 4089af5929e..6f772deee2d 100644 --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -217,7 +217,7 @@ init_secondary(int cpu) CTR0(KTR_SMP, "go into scheduler"); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index b42f65b9e39..4eebfe21993 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -293,7 +293,7 @@ init_secondary(uint64_t cpu) MPASS(PCPU_GET(curpcb) == NULL); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 6ba41eb80dc..25f8bfc04d4 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -1662,12 +1662,22 @@ sched_idletd(void *dummy) } } +static void +sched_throw_tail(struct thread *td) +{ + + mtx_assert(&sched_lock, MA_OWNED); + KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count")); + cpu_throw(td, choosethread()); /* doesn't return */ +} + /* - * A CPU is entering for the first time or a thread is exiting. + * A CPU is entering for the first time. */ void -sched_throw(struct thread *td) +sched_ap_entry(void) { + /* * Correct spinlock nesting. The idle thread context that we are * borrowing was created so that it would start out with a single @@ -1677,20 +1687,29 @@ sched_throw(struct thread *td) * spinlock_exit() will simply adjust the counts without allowing * spin lock using code to interrupt us. */ - if (td == NULL) { - mtx_lock_spin(&sched_lock); - spinlock_exit(); - PCPU_SET(switchtime, cpu_ticks()); - PCPU_SET(switchticks, ticks); - } else { - lock_profile_release_lock(&sched_lock.lock_object, true); - MPASS(td->td_lock == &sched_lock); - td->td_lastcpu = td->td_oncpu; - td->td_oncpu = NOCPU; - } - mtx_assert(&sched_lock, MA_OWNED); - KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count")); - cpu_throw(td, choosethread()); /* doesn't return */ + mtx_lock_spin(&sched_lock); + spinlock_exit(); + PCPU_SET(switchtime, cpu_ticks()); + PCPU_SET(switchticks, ticks); + + sched_throw_tail(NULL); +} + +/* + * A thread is exiting. + */ +void +sched_throw(struct thread *td) +{ + + MPASS(td != NULL); + MPASS(td->td_lock == &sched_lock); + + lock_profile_release_lock(&sched_lock.lock_object, true); + td->td_lastcpu = td->td_oncpu; + td->td_oncpu = NOCPU; + + sched_throw_tail(td); } void diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index ce7ce4cd2bd..e311c8da8ba 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -2985,7 +2985,52 @@ sched_idletd(void *dummy) } /* - * A CPU is entering for the first time or a thread is exiting. + * sched_throw_grab() chooses a thread from the queue to switch to + * next. It returns with the tdq lock dropped in a spinlock section to + * keep interrupts disabled until the CPU is running in a proper threaded + * context. + */ +static struct thread * +sched_throw_grab(struct tdq *tdq) +{ + struct thread *newtd; + + newtd = choosethread(); + spinlock_enter(); + TDQ_UNLOCK(tdq); + KASSERT(curthread->td_md.md_spinlock_count == 1, + ("invalid count %d", curthread->td_md.md_spinlock_count)); + return (newtd); +} + +/* + * A CPU is entering for the first time. + */ +void +sched_ap_entry(void) +{ + struct thread *newtd; + struct tdq *tdq; + + tdq = TDQ_SELF(); + + /* This should have been setup in schedinit_ap(). */ + THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq)); + + TDQ_LOCK(tdq); + /* Correct spinlock nesting. */ + spinlock_exit(); + PCPU_SET(switchtime, cpu_ticks()); + PCPU_SET(switchticks, ticks); + + newtd = sched_throw_grab(tdq); + + /* doesn't return */ + cpu_throw(NULL, newtd); +} + +/* + * A thread is exiting. */ void sched_throw(struct thread *td) @@ -2994,30 +3039,20 @@ sched_throw(struct thread *td) struct tdq *tdq; tdq = TDQ_SELF(); - if (__predict_false(td == NULL)) { - TDQ_LOCK(tdq); - /* Correct spinlock nesting. */ - spinlock_exit(); - PCPU_SET(switchtime, cpu_ticks()); - PCPU_SET(switchticks, ticks); - } else { - THREAD_LOCK_ASSERT(td, MA_OWNED); - THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq)); - tdq_load_rem(tdq, td); - td->td_lastcpu = td->td_oncpu; - td->td_oncpu = NOCPU; - thread_lock_block(td); - } - newtd = choosethread(); - spinlock_enter(); - TDQ_UNLOCK(tdq); - KASSERT(curthread->td_md.md_spinlock_count == 1, - ("invalid count %d", curthread->td_md.md_spinlock_count)); + + MPASS(td != NULL); + THREAD_LOCK_ASSERT(td, MA_OWNED); + THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq)); + + tdq_load_rem(tdq, td); + td->td_lastcpu = td->td_oncpu; + td->td_oncpu = NOCPU; + thread_lock_block(td); + + newtd = sched_throw_grab(tdq); + /* doesn't return */ - if (__predict_false(td == NULL)) - cpu_throw(td, newtd); /* doesn't return */ - else - cpu_switch(td, newtd, TDQ_LOCKPTR(tdq)); + cpu_switch(td, newtd, TDQ_LOCKPTR(tdq)); } /* diff --git a/sys/mips/mips/mp_machdep.c b/sys/mips/mips/mp_machdep.c index dc089db1d18..2582c2b65e7 100644 --- a/sys/mips/mips/mp_machdep.c +++ b/sys/mips/mips/mp_machdep.c @@ -335,7 +335,7 @@ smp_init_secondary(u_int32_t cpuid) cpu_initclocks_ap(); /* enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ diff --git a/sys/powerpc/powerpc/mp_machdep.c b/sys/powerpc/powerpc/mp_machdep.c index 627cde77adb..33ef870b818 100644 --- a/sys/powerpc/powerpc/mp_machdep.c +++ b/sys/powerpc/powerpc/mp_machdep.c @@ -112,7 +112,7 @@ machdep_ap_bootstrap(void) cpu_initclocks_ap(); /* Announce ourselves awake, and enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); } void diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c index 57d5606a3b8..74647b8fcba 100644 --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -291,7 +291,7 @@ init_secondary(uint64_t hart) MPASS(PCPU_GET(curpcb) == NULL); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ diff --git a/sys/sys/sched.h b/sys/sys/sched.h index 8041a2bc12d..a9598767e4c 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -91,6 +91,7 @@ void sched_nice(struct proc *p, int nice); * Threads are switched in and out, block on resources, have temporary * priorities inherited from their procs, and use up cpu time. */ +void sched_ap_entry(void); void sched_exit_thread(struct thread *td, struct thread *child); u_int sched_estcpu(struct thread *td); void sched_fork_thread(struct thread *td, struct thread *child); diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 1fac244cbed..7a72c501ff2 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -1099,7 +1099,7 @@ init_secondary_tail(void) */ MPASS(PCPU_GET(curpcb) == NULL); - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */