From 7e1f6dfe9d7ac65419d57b36dee19cd1a3e996f5 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 18 Dec 2001 00:27:18 +0000 Subject: [PATCH] Modify the critical section API as follows: - The MD functions critical_enter/exit are renamed to start with a cpu_ prefix. - MI wrapper functions critical_enter/exit maintain a per-thread nesting count and a per-thread critical section saved state set when entering a critical section while at nesting level 0 and restored when exiting to nesting level 0. This moves the saved state out of spin mutexes so that interlocking spin mutexes works properly. - Most low-level MD code that used critical_enter/exit now use cpu_critical_enter/exit. MI code such as device drivers and spin mutexes use the MI wrappers. Note that since the MI wrappers store the state in the current thread, they do not have any return values or arguments. - mtx_intr_enable() is replaced with a constant CRITICAL_FORK which is assigned to curthread->td_savecrit during fork_exit(). Tested on: i386, alpha --- sys/alpha/alpha/db_interface.c | 4 +- sys/alpha/alpha/genassym.c | 1 - sys/alpha/alpha/interrupt.c | 4 +- sys/alpha/alpha/machdep.c | 16 ++-- sys/alpha/alpha/prom.c | 6 +- sys/alpha/alpha/trap.c | 8 +- sys/alpha/include/cpufunc.h | 6 +- sys/alpha/include/mutex.h | 2 - sys/alpha/pci/cia.c | 8 +- sys/alpha/pci/t2.c | 4 +- sys/alpha/pci/t2_pci.c | 4 +- sys/amd64/amd64/db_interface.c | 4 +- sys/amd64/amd64/fpu.c | 16 ++-- sys/amd64/amd64/genassym.c | 1 - sys/amd64/amd64/initcpu.c | 4 +- sys/amd64/amd64/sys_machdep.c | 4 +- sys/amd64/amd64/vm_machdep.c | 4 +- sys/amd64/include/cpufunc.h | 11 ++- sys/amd64/include/mutex.h | 50 ++---------- sys/amd64/isa/npx.c | 16 ++-- sys/dev/bktr/bktr_os.h | 6 +- sys/dev/cy/cy.c | 127 +++++++++++++---------------- sys/dev/cy/cy_isa.c | 127 +++++++++++++---------------- sys/dev/sound/isa/mpu.c | 11 ++- sys/i386/i386/db_interface.c | 4 +- sys/i386/i386/genassym.c | 1 - sys/i386/i386/initcpu.c | 4 +- sys/i386/i386/perfmon.c | 12 +-- sys/i386/i386/sys_machdep.c | 4 +- sys/i386/i386/vm_machdep.c | 4 +- sys/i386/include/cpufunc.h | 11 ++- sys/i386/include/mutex.h | 50 ++---------- sys/i386/isa/cy.c | 127 +++++++++++++---------------- sys/i386/isa/npx.c | 16 ++-- sys/ia64/ia64/db_interface.c | 4 +- sys/ia64/ia64/genassym.c | 1 - sys/ia64/ia64/pmap.c | 4 +- sys/ia64/ia64/sapic.c | 8 +- sys/ia64/include/cpufunc.h | 6 +- sys/ia64/include/mutex.h | 12 +-- sys/ia64/include/profile.h | 4 +- sys/kern/kern_fork.c | 9 +- sys/kern/kern_idle.c | 4 +- sys/kern/kern_ktr.c | 6 +- sys/kern/kern_mutex.c | 8 +- sys/kern/kern_switch.c | 25 ++++++ sys/kern/kern_synch.c | 3 - sys/kern/subr_prof.c | 10 +-- sys/kern/subr_trap.c | 8 +- sys/kern/subr_turnstile.c | 8 +- sys/kern/subr_witness.c | 13 ++- sys/powerpc/include/cpufunc.h | 6 +- sys/powerpc/include/mutex.h | 17 +--- sys/powerpc/powerpc/genassym.c | 1 - sys/sparc64/include/cpufunc.h | 6 +- sys/sparc64/include/mutex.h | 7 -- sys/sparc64/sparc64/intr_machdep.c | 4 +- sys/sys/_mutex.h | 1 - sys/sys/mutex.h | 22 +++-- sys/sys/proc.h | 2 + sys/sys/systm.h | 2 + 61 files changed, 376 insertions(+), 502 deletions(-) diff --git a/sys/alpha/alpha/db_interface.c b/sys/alpha/alpha/db_interface.c index fea763d85de..041b0b9285f 100644 --- a/sys/alpha/alpha/db_interface.c +++ b/sys/alpha/alpha/db_interface.c @@ -192,7 +192,7 @@ kdb_trap(a0, a1, a2, entry, regs) ddb_regs = *regs; - s = critical_enter(); + s = cpu_critical_enter(); #ifdef SMP #ifdef DIAGNOSTIC @@ -219,7 +219,7 @@ kdb_trap(a0, a1, a2, entry, regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/alpha/alpha/genassym.c b/sys/alpha/alpha/genassym.c index 8d68d73d228..e9896753e73 100644 --- a/sys/alpha/alpha/genassym.c +++ b/sys/alpha/alpha/genassym.c @@ -76,7 +76,6 @@ ASSYM(PC_IDLEPCBPHYS, offsetof(struct pcpu, pc_idlepcbphys)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c index 29bb74f54e4..31438566050 100644 --- a/sys/alpha/alpha/interrupt.c +++ b/sys/alpha/alpha/interrupt.c @@ -99,13 +99,13 @@ interrupt(a0, a1, a2, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif atomic_add_int(&td->td_intr_nesting_level, 1); #ifndef KSTACK_GUARD diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index ef763a9ad4a..a6f181c7a48 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -2007,11 +2007,11 @@ alpha_fpstate_check(struct thread *td) #ifndef SMP critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td->td_pcb->pcb_hw.apcb_flags & ALPHA_PCB_FLAGS_FEN) if (td != PCPU_GET(fpcurthread)) panic("alpha_check_fpcurthread: bogus"); - critical_exit(s); + cpu_critical_exit(s); #endif } @@ -2033,7 +2033,7 @@ alpha_fpstate_save(struct thread *td, int write) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td != NULL && td == PCPU_GET(fpcurthread)) { /* * If curthread != fpcurthread, then we need to enable FEN @@ -2068,7 +2068,7 @@ alpha_fpstate_save(struct thread *td, int write) alpha_pal_wrfen(0); } } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2081,7 +2081,7 @@ alpha_fpstate_drop(struct thread *td) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) { if (td == curthread) { /* @@ -2097,7 +2097,7 @@ alpha_fpstate_drop(struct thread *td) } PCPU_SET(fpcurthread, NULL); } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2112,7 +2112,7 @@ alpha_fpstate_switch(struct thread *td) /* * Enable FEN so that we can access the fp registers. */ - s = critical_enter(); + s = cpu_critical_enter(); alpha_pal_wrfen(1); if (PCPU_GET(fpcurthread)) { /* @@ -2139,7 +2139,7 @@ alpha_fpstate_switch(struct thread *td) } td->td_md.md_flags |= MDP_FPUSED; - critical_exit(s); + cpu_critical_exit(s); } /* diff --git a/sys/alpha/alpha/prom.c b/sys/alpha/alpha/prom.c index 54d913a4df5..d4dd26766cc 100644 --- a/sys/alpha/alpha/prom.c +++ b/sys/alpha/alpha/prom.c @@ -193,7 +193,7 @@ enter_prom() pt_entry_t *lev1map; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (!prom_mapped) { #ifdef SIMOS @@ -232,7 +232,7 @@ leave_prom(s) lev1map[0] = saved_pte[0]; /* XXX */ prom_cache_sync(); /* XXX */ } - critical_exit(s); + cpu_critical_exit(s); } static void @@ -272,7 +272,7 @@ prom_halt(halt) /* * Turn off interrupts, for sanity. */ - critical_enter(); + cpu_critical_enter(); /* * Set "boot request" part of the CPU state depending on what diff --git a/sys/alpha/alpha/trap.c b/sys/alpha/alpha/trap.c index 4c7694dd712..99278faf759 100644 --- a/sys/alpha/alpha/trap.c +++ b/sys/alpha/alpha/trap.c @@ -270,13 +270,13 @@ trap(a0, a1, a2, entry, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; @@ -672,13 +672,13 @@ syscall(code, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; diff --git a/sys/alpha/include/cpufunc.h b/sys/alpha/include/cpufunc.h index 51a1fdcb99b..dbee9115e03 100644 --- a/sys/alpha/include/cpufunc.h +++ b/sys/alpha/include/cpufunc.h @@ -35,6 +35,8 @@ #include #include +#define CRITICAL_FORK (ALPHA_PSL_IPL_0) + #ifdef __GNUC__ static __inline void @@ -46,13 +48,13 @@ breakpoint(void) #endif static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return (alpha_pal_swpipl(ALPHA_PSL_IPL_MCES)); } static __inline void -critical_exit(critical_t ipl) +cpu_critical_exit(critical_t ipl) { alpha_pal_swpipl(ipl); } diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h index 970049bda0b..7300c08b15f 100644 --- a/sys/alpha/include/mutex.h +++ b/sys/alpha/include/mutex.h @@ -39,8 +39,6 @@ /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit = ALPHA_PSL_IPL_0; while (0) - #endif /* _KERNEL */ #else /* !LOCORE */ diff --git a/sys/alpha/pci/cia.c b/sys/alpha/pci/cia.c index c2bcafce51a..09436e33c4e 100644 --- a/sys/alpha/pci/cia.c +++ b/sys/alpha/pci/cia.c @@ -156,12 +156,12 @@ cia_swiz_set_hae_mem(void *arg, u_int32_t pa) u_int32_t msb = pa & REG1; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); cia_hae_mem = (cia_hae_mem & ~REG1) | msb; REGVAL(CIA_CSR_HAE_MEM) = cia_hae_mem; alpha_mb(); cia_hae_mem = REGVAL(CIA_CSR_HAE_MEM); - critical_exit(s); + cpu_critical_exit(s); } return pa & ~REG1; } @@ -226,7 +226,7 @@ cia_sgmap_invalidate_pyxis(void) int i; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); /* * Put the Pyxis into PCI loopback mode. @@ -257,7 +257,7 @@ cia_sgmap_invalidate_pyxis(void) REGVAL(CIA_CSR_CTRL) = ctrl; alpha_mb(); - critical_exit(s); + cpu_critical_exit(s); } static void diff --git a/sys/alpha/pci/t2.c b/sys/alpha/pci/t2.c index c8ade0c8e8c..d9d0b9d0ada 100644 --- a/sys/alpha/pci/t2.c +++ b/sys/alpha/pci/t2.c @@ -118,14 +118,14 @@ t2_set_hae_mem(void *arg, u_int32_t pa) msb = pa & 0xf8000000; pa -= msb; msb >>= 27; /* t2 puts high bits in the bottom of the register */ - s = critical_enter(); + s = cpu_critical_enter(); if (msb != t2_hae_mem[hose]) { t2_hae_mem[hose] = msb; t2_csr[hose]->hae0_1 = t2_hae_mem[hose]; alpha_mb(); t2_hae_mem[hose] = t2_csr[hose]->hae0_1; } - critical_exit(s); + cpu_critical_exit(s); } return pa; } diff --git a/sys/alpha/pci/t2_pci.c b/sys/alpha/pci/t2_pci.c index 87a3dca98d8..b50555fb93d 100644 --- a/sys/alpha/pci/t2_pci.c +++ b/sys/alpha/pci/t2_pci.c @@ -92,7 +92,7 @@ t2_pcib_maxslots(device_t dev) #define T2_TYPE1_SETUP(b,s,old_hae3) if((b)) { \ do { \ - (s) = critical_enter(); \ + (s) = cpu_critical_enter(); \ (old_hae3) = REGVAL(T2_HAE0_3); \ alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3) | (1<<30); \ @@ -105,7 +105,7 @@ t2_pcib_maxslots(device_t dev) alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3); \ alpha_mb(); \ - critical_exit((s)); \ + cpu_critical_exit((s)); \ } while(0); \ } diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index 9708cad042f..000085b0e66 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index ff8c4cf20fb..34a822a230a 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 59f314766c5..99fa50259bb 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c index b32c786851b..8a9a2a5224d 100644 --- a/sys/amd64/amd64/initcpu.c +++ b/sys/amd64/amd64/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c index a88acc98b8b..b7beea96f58 100644 --- a/sys/amd64/amd64/sys_machdep.c +++ b/sys/amd64/amd64/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 322b5e73cfd..4f1dab24d61 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 4a9300afd93..a4f57cb8107 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include +#include __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index ae37b234b21..e68bce7d441 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c index ff8c4cf20fb..34a822a230a 100644 --- a/sys/amd64/isa/npx.c +++ b/sys/amd64/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/dev/bktr/bktr_os.h b/sys/dev/bktr/bktr_os.h index 21d3f602e38..174072321c4 100644 --- a/sys/dev/bktr/bktr_os.h +++ b/sys/dev/bktr/bktr_os.h @@ -61,9 +61,9 @@ void free_bktr_mem(bktr_ptr_t, bus_dmamap_t, vm_offset_t); /************************************/ #if defined(__FreeBSD__) #if (__FreeBSD_version >=500000) -#define DECLARE_INTR_MASK(s) critical_t s -#define DISABLE_INTR(s) s = critical_enter() -#define ENABLE_INTR(s) critical_exit(s) +#define DECLARE_INTR_MASK(s) /* no need to declare 's' */ +#define DISABLE_INTR(s) critical_enter() +#define ENABLE_INTR(s) critical_exit() #else #define DECLARE_INTR_MASK(s) intrmask_t s #define DISABLE_INTR(s) s=spltty() diff --git a/sys/dev/cy/cy.c b/sys/dev/cy/cy.c index 20316adcd7c..0510e729417 100644 --- a/sys/dev/cy/cy.c +++ b/sys/dev/cy/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/cy/cy_isa.c b/sys/dev/cy/cy_isa.c index 20316adcd7c..0510e729417 100644 --- a/sys/dev/cy/cy_isa.c +++ b/sys/dev/cy/cy_isa.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/sound/isa/mpu.c b/sys/dev/sound/isa/mpu.c index c0d305e634f..ed9b74612ff 100644 --- a/sys/dev/sound/isa/mpu.c +++ b/sys/dev/sound/isa/mpu.c @@ -241,7 +241,6 @@ mpu_probe2(device_t dev) sc_p scp; int unit, i; intrmask_t irqp0, irqp1; - critical_t savecrit; scp = device_get_softc(dev); unit = device_get_unit(dev); @@ -267,7 +266,7 @@ mpu_probe2(device_t dev) * Idea-stolen-from: sys/isa/sio.c:sioprobe() */ - savecrit = critical_enter(); + critical_enter(); /* * See the initial irq. We have to do this now, @@ -279,7 +278,7 @@ mpu_probe2(device_t dev) /* Switch to uart mode. */ if (mpu_uartmode(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: mode switching failed.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -298,7 +297,7 @@ mpu_probe2(device_t dev) break; } if (irqp1 == irqp0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: switching the mode gave no interrupt.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -307,13 +306,13 @@ mpu_probe2(device_t dev) no_irq: /* Wait to see an ACK. */ if (mpu_waitack(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: not acked.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); } - critical_exit(savecrit); + critical_exit(); if (device_get_flags(dev) & MPU_DF_NO_IRQ) scp->irq_val = 0; diff --git a/sys/i386/i386/db_interface.c b/sys/i386/i386/db_interface.c index 9708cad042f..000085b0e66 100644 --- a/sys/i386/i386/db_interface.c +++ b/sys/i386/i386/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 59f314766c5..99fa50259bb 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c index b32c786851b..8a9a2a5224d 100644 --- a/sys/i386/i386/initcpu.c +++ b/sys/i386/i386/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/i386/i386/perfmon.c b/sys/i386/i386/perfmon.c index ed0b53ef4da..4d6c9a2e617 100644 --- a/sys/i386/i386/perfmon.c +++ b/sys/i386/i386/perfmon.c @@ -124,11 +124,11 @@ perfmon_setup(int pmc, unsigned int control) perfmon_inuse |= (1 << pmc); control &= ~(PMCF_SYS_FLAGS << 16); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] = control; writectl(pmc); wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } @@ -169,11 +169,11 @@ perfmon_start(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] |= (PMCF_EN << 16); wrmsr(msr_pmc[pmc], pmc_shadow[pmc]); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; @@ -188,11 +188,11 @@ perfmon_stop(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); pmc_shadow[pmc] = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; ctl_shadow[pmc] &= ~(PMCF_EN << 16); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c index a88acc98b8b..b7beea96f58 100644 --- a/sys/i386/i386/sys_machdep.c +++ b/sys/i386/i386/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 322b5e73cfd..4f1dab24d61 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 4a9300afd93..a4f57cb8107 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include +#include __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h index ae37b234b21..e68bce7d441 100644 --- a/sys/i386/include/mutex.h +++ b/sys/i386/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/i386/isa/cy.c b/sys/i386/isa/cy.c index 20316adcd7c..0510e729417 100644 --- a/sys/i386/isa/cy.c +++ b/sys/i386/isa/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index ff8c4cf20fb..34a822a230a 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/ia64/ia64/db_interface.c b/sys/ia64/ia64/db_interface.c index 9a254da9a65..3d576cd5a0c 100644 --- a/sys/ia64/ia64/db_interface.c +++ b/sys/ia64/ia64/db_interface.c @@ -336,7 +336,7 @@ kdb_trap(int vector, struct trapframe *regs) __asm __volatile("flushrs"); /* so we can look at them */ - s = critical_enter(); + s = cpu_critical_enter(); #if 0 db_printf("stopping %x\n", PCPU_GET(other_cpus)); @@ -359,7 +359,7 @@ kdb_trap(int vector, struct trapframe *regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c index fbe60e586e2..5370da02652 100644 --- a/sys/ia64/ia64/genassym.c +++ b/sys/ia64/ia64/genassym.c @@ -71,7 +71,6 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 34b129d06a8..a144db6df0b 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -577,7 +577,7 @@ pmap_invalidate_all(pmap_t pmap) KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), ("invalidating TLB for non-current pmap")); - psr = critical_enter(); + psr = cpu_critical_enter(); addr = pmap_ptc_e_base; for (i = 0; i < pmap_ptc_e_count1; i++) { for (j = 0; j < pmap_ptc_e_count2; j++) { @@ -586,7 +586,7 @@ pmap_invalidate_all(pmap_t pmap) } addr += pmap_ptc_e_stride1; } - critical_exit(psr); + cpu_critical_exit(psr); } static u_int32_t diff --git a/sys/ia64/ia64/sapic.c b/sys/ia64/ia64/sapic.c index 6af5cc08775..e13b8d598e8 100644 --- a/sys/ia64/ia64/sapic.c +++ b/sys/ia64/ia64/sapic.c @@ -82,10 +82,10 @@ sapic_read_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); p[0] = sapic_read(sa, SAPIC_RTE_BASE + 2*which); p[1] = sapic_read(sa, SAPIC_RTE_BASE + 2*which + 1); - critical_exit(c); + cpu_critical_exit(c); } #endif @@ -96,10 +96,10 @@ sapic_write_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); sapic_write(sa, SAPIC_RTE_BASE + 2*which, p[0]); sapic_write(sa, SAPIC_RTE_BASE + 2*which + 1, p[1]); - critical_exit(c); + cpu_critical_exit(c); } struct sapic * diff --git a/sys/ia64/include/cpufunc.h b/sys/ia64/include/cpufunc.h index 45390e9b974..c2d074fc363 100644 --- a/sys/ia64/include/cpufunc.h +++ b/sys/ia64/include/cpufunc.h @@ -34,6 +34,8 @@ #include #include +#define CRITICAL_FORK (ia64_get_psr() |= IA64_PSR_I) + #ifdef __GNUC__ static __inline void @@ -283,7 +285,7 @@ enable_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t psr; @@ -293,7 +295,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t psr) +cpu_critical_exit(critical_t psr) { __asm __volatile ("mov psr.l=%0;; srlz.d" :: "r" (psr)); } diff --git a/sys/ia64/include/mutex.h b/sys/ia64/include/mutex.h index b14d7aef4f3..96d8f3d48a8 100644 --- a/sys/ia64/include/mutex.h +++ b/sys/ia64/include/mutex.h @@ -32,17 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#include - -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= IA64_PSR_I; while (0) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/ia64/include/profile.h b/sys/ia64/include/profile.h index 038c8057b54..8d93c5c27ac 100644 --- a/sys/ia64/include/profile.h +++ b/sys/ia64/include/profile.h @@ -100,9 +100,9 @@ _mcount: \n\ * The following two macros do splhigh and splx respectively. */ #define MCOUNT_ENTER(s) \n\ - _c = critical_enter() + _c = cpu_critical_enter() #define MCOUNT_EXIT(s) \n\ - (void)critical_exit(_c) + cpu_critical_exit(_c) #define MCOUNT_DECL(s) critical_t c; #ifdef GUPROF struct gmonparam; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index bb52a34b8ef..bc03078976f 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -467,9 +467,6 @@ again: if (p1->p_sflag & PS_PROFIL) startprofclock(p2); mtx_unlock_spin(&sched_lock); - /* - * We start off holding one spinlock after fork: sched_lock. - */ PROC_LOCK(p1); p2->p_ucred = crhold(p1->p_ucred); p2->p_thread.td_ucred = crhold(p2->p_ucred); /* XXXKSE */ @@ -766,10 +763,8 @@ fork_exit(callout, arg, frame) */ sched_lock.mtx_lock = (uintptr_t)td; sched_lock.mtx_recurse = 0; - /* - * XXX: We really shouldn't have to do this. - */ - mtx_intr_enable(&sched_lock); + td->td_critnest = 1; + td->td_savecrit = CRITICAL_FORK; CTR3(KTR_PROC, "fork_exit: new proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); if (PCPU_GET(switchtime.tv_sec) == 0) diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c index 92e5cb38b83..4dee96d68a5 100644 --- a/sys/kern/kern_idle.c +++ b/sys/kern/kern_idle.c @@ -47,8 +47,10 @@ idle_setup(void *dummy) error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle: cpu%d", pc->pc_cpuid); pc->pc_idlethread = &p->p_thread; - if (pc->pc_curthread == NULL) + if (pc->pc_curthread == NULL) { pc->pc_curthread = pc->pc_idlethread; + pc->pc_idlethread->td_critnest = 0; + } #else error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle"); diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c index 309d6e31bb1..596a724ff4a 100644 --- a/sys/kern/kern_ktr.c +++ b/sys/kern/kern_ktr.c @@ -133,9 +133,9 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, td = curthread; if (td->td_inktr) return; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (((1 << KTR_CPU) & ktr_cpumask) == 0) { - critical_exit(savecrit); + cpu_critical_exit(savecrit); return; } td->td_inktr++; @@ -145,7 +145,7 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, } while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0); entry = &ktr_buf[saveindex]; entry->ktr_cpu = KTR_CPU; - critical_exit(savecrit); + cpu_critical_exit(savecrit); nanotime(&entry->ktr_tv); #ifdef KTR_EXTEND entry->ktr_filename = filename; diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index d4809af17f1..852b570ae4d 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index 65054dfe002..1a2afa44ac5 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -69,6 +69,31 @@ setrunqueue(struct thread *td) runq_add(&runq, td->td_kse); } +/* Critical sections that prevent preemption. */ +void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 0) + td->td_savecrit = cpu_critical_enter(); + td->td_critnest++; +} + +void +critical_exit(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 1) { + td->td_critnest = 0; + cpu_critical_exit(td->td_savecrit); + } else + td->td_critnest--; +} + /* * Clear the status bit of the queue corresponding to priority level pri, * indicating that it is empty. diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 7034c0a8dc6..fce470f67d4 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -698,7 +698,6 @@ mi_switch() #if 0 register struct rlimit *rlim; #endif - critical_t sched_crit; u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); @@ -773,14 +772,12 @@ mi_switch() PCPU_SET(switchtime, new_switchtime); CTR3(KTR_PROC, "mi_switch: old proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); - sched_crit = sched_lock.mtx_savecrit; sched_nest = sched_lock.mtx_recurse; td->td_lastcpu = td->td_kse->ke_oncpu; td->td_kse->ke_oncpu = NOCPU; td->td_kse->ke_flags &= ~KEF_NEEDRESCHED; cpu_switch(); td->td_kse->ke_oncpu = PCPU_GET(cpuid); - sched_lock.mtx_savecrit = sched_crit; sched_lock.mtx_recurse = sched_nest; sched_lock.mtx_lock = (uintptr_t)td; CTR3(KTR_PROC, "mi_switch: new proc %p (pid %d, %s)", p, p->p_pid, diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c index 67629d4c0c0..88086515762 100644 --- a/sys/kern/subr_prof.c +++ b/sys/kern/subr_prof.c @@ -90,7 +90,6 @@ kmupetext(uintfptr_t nhighpc) struct gmonparam np; /* slightly large */ struct gmonparam *p = &_gmonparam; char *cp; - critical_t savecrit; GIANT_REQUIRED; bcopy(p, &np, sizeof(*p)); @@ -127,7 +126,7 @@ kmupetext(uintfptr_t nhighpc) np.mcount_count = &KCOUNT(&np, PC_TO_I(&np, mcount)); np.mexitcount_count = &KCOUNT(&np, PC_TO_I(&np, mexitcount)); #endif - savecrit = critical_enter(); + critical_enter(); bcopy(p->tos, np.tos, p->tossize); bzero((char *)np.tos + p->tossize, np.tossize - p->tossize); bcopy(p->kcount, np.kcount, p->kcountsize); @@ -137,7 +136,7 @@ kmupetext(uintfptr_t nhighpc) bzero((char *)np.froms + p->fromssize, np.fromssize - p->fromssize); cp = (char *)p->tos; bcopy(&np, p, sizeof(*p)); - critical_exit(savecrit); + critical_exit(); free(cp, M_GPROF); } @@ -156,7 +155,6 @@ kmstartup(dummy) int nullfunc_loop_overhead; int nullfunc_loop_profiled_time; uintfptr_t tmp_addr; - critical_t savecrit; #endif /* @@ -195,7 +193,7 @@ kmstartup(dummy) * Disable interrupts to avoid interference while we calibrate * things. */ - savecrit = critical_enter(); + critical_enter(); /* * Determine overheads. @@ -249,7 +247,7 @@ kmstartup(dummy) p->state = GMON_PROF_OFF; stopguprof(p); - critical_exit(savecrit); + critical_exit(); nullfunc_loop_profiled_time = 0; for (tmp_addr = (uintfptr_t)nullfunc_loop_profiled; diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index a8995766324..6f17f8fcee3 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -141,9 +141,9 @@ ast(framep) panic("Returning to user mode with mutex(s) held"); #endif mtx_assert(&Giant, MA_NOTOWNED); - s = critical_enter(); + s = cpu_critical_enter(); while ((ke->ke_flags & (KEF_ASTPENDING | KEF_NEEDRESCHED)) != 0) { - critical_exit(s); + cpu_critical_exit(s); td->td_frame = framep; /* * This updates the p_sflag's for the checks below in one @@ -195,13 +195,13 @@ ast(framep) crfree(td->td_ucred); mtx_unlock(&Giant); td->td_ucred = NULL; - s = critical_enter(); + s = cpu_critical_enter(); } mtx_assert(&Giant, MA_NOTOWNED); /* * We need to keep interrupts disabled so that if any further AST's * come in, the interrupt they come in on will be delayed until we * finish returning to userland. We assume that the return to userland - * will perform the equivalent of critical_exit(). + * will perform the equivalent of cpu_critical_exit(). */ } diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index d4809af17f1..852b570ae4d 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 454fd55c76e..5e4dc61c6ef 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -252,7 +252,6 @@ static struct mtx all_mtx = { { NULL }, /* mtx_object.lo_list */ NULL }, /* mtx_object.lo_witness */ MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */ - 0, /* mtx_savecrit */ TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL } /* mtx_contested */ }; @@ -836,7 +835,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) instance->li_flags--; goto out; } - s = critical_enter(); + s = cpu_critical_enter(); CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__, td->td_proc->p_pid, @@ -846,7 +845,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) for (j = i; j < (*lock_list)->ll_count; j++) (*lock_list)->ll_children[j] = (*lock_list)->ll_children[j + 1]; - critical_exit(s); + cpu_critical_exit(s); if ((*lock_list)->ll_count == 0) { lle = *lock_list; *lock_list = lle->ll_next; @@ -896,7 +895,7 @@ witness_sleep(int check_only, struct lock_object *lock, const char *file, /* * Preemption bad because we need PCPU_PTR(spinlocks) to not change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); td = curthread; lock_list = &td->td_sleeplocks; again: @@ -931,7 +930,7 @@ again: if (witness_ddb && n) Debugger(__func__); #endif /* DDB */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (n); } @@ -1360,9 +1359,9 @@ witness_list(struct thread *td) * Preemption bad because we need PCPU_PTR(spinlocks) to not * change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); nheld += witness_list_locks(PCPU_PTR(spinlocks)); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } return (nheld); } diff --git a/sys/powerpc/include/cpufunc.h b/sys/powerpc/include/cpufunc.h index 0fd0dd0224a..25f37d38874 100644 --- a/sys/powerpc/include/cpufunc.h +++ b/sys/powerpc/include/cpufunc.h @@ -35,6 +35,8 @@ #include +#define CRITICAL_FORK (mfmsr() |= PSL_EE) + #ifdef __GNUC__ static __inline void @@ -114,7 +116,7 @@ save_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return ((critical_t)save_intr()); @@ -128,7 +130,7 @@ restore_intr(unsigned int msr) } static __inline void -critical_exit(critical_t msr) +cpu_critical_exit(critical_t msr) { return (restore_intr((unsigned int)msr)); diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h index 0aeaa7488ca..0b7fe2633e2 100644 --- a/sys/powerpc/include/mutex.h +++ b/sys/powerpc/include/mutex.h @@ -32,22 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_EE; while (0) - -/* - * Assembly macros (for internal use only) - *-------------------------------------------------------------------------- - */ - -#define _V(x) __STRING(x) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c index 7ab6e777172..c212c6bc15a 100644 --- a/sys/powerpc/powerpc/genassym.c +++ b/sys/powerpc/powerpc/genassym.c @@ -66,7 +66,6 @@ ASSYM(PC_SWITCHTIME, offsetof(struct pcpu, pc_switchtime)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(PM_KERNELSR, offsetof(struct pmap, pm_sr[KERNEL_SR])); ASSYM(PM_USERSR, offsetof(struct pmap, pm_sr[USER_SR])); diff --git a/sys/sparc64/include/cpufunc.h b/sys/sparc64/include/cpufunc.h index 18f06b5c325..f9739ad234d 100644 --- a/sys/sparc64/include/cpufunc.h +++ b/sys/sparc64/include/cpufunc.h @@ -156,6 +156,8 @@ STNC_GEN(u_long, stxa); : : "r" (val), "rI" (xor)); \ } while (0) +#define CRITICAL_FORK (0) + static __inline void breakpoint(void) { @@ -163,7 +165,7 @@ breakpoint(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t pil; @@ -173,7 +175,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t pil) +cpu_critical_exit(critical_t pil) { wrpr(pil, pil, 0); } diff --git a/sys/sparc64/include/mutex.h b/sys/sparc64/include/mutex.h index 293f8d67e25..c9b2e1ddde0 100644 --- a/sys/sparc64/include/mutex.h +++ b/sys/sparc64/include/mutex.h @@ -29,11 +29,4 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -static __inline void -mtx_intr_enable(struct mtx *mtx) -{ - - mtx->mtx_savecrit = 0; -} - #endif /* !_MACHINE_MUTEX_H_ */ diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c index 3b39f9a29fb..ca28a60f4f5 100644 --- a/sys/sparc64/sparc64/intr_machdep.c +++ b/sys/sparc64/sparc64/intr_machdep.c @@ -99,7 +99,7 @@ intr_dequeue(struct trapframe *tf) u_long next; u_long tail; - crit = critical_enter(); + crit = cpu_critical_enter(); iq = PCPU_PTR(iq); for (head = iq->iq_head;; head = next) { for (tail = iq->iq_tail; tail != head;) { @@ -115,7 +115,7 @@ intr_dequeue(struct trapframe *tf) if (head == next) break; } - critical_exit(crit); + cpu_critical_exit(crit); } void diff --git a/sys/sys/_mutex.h b/sys/sys/_mutex.h index 82f3648a1b5..ef5de769d5f 100644 --- a/sys/sys/_mutex.h +++ b/sys/sys/_mutex.h @@ -39,7 +39,6 @@ struct mtx { struct lock_object mtx_object; /* Common lock properties. */ volatile uintptr_t mtx_lock; /* owner (and state for sleep locks) */ volatile u_int mtx_recurse; /* number of recursive holds */ - critical_t mtx_savecrit; /* saved flags (for spin locks) */ TAILQ_HEAD(, thread) mtx_blocked; /* threads blocked on this lock */ LIST_ENTRY(mtx) mtx_contested; /* list of all contested locks */ }; diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index d581973a24c..a4eb0184674 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -101,8 +101,7 @@ void mtx_init(struct mtx *m, const char *description, int opts); void mtx_destroy(struct mtx *m); void _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line); -void _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, - const char *file, int line); +void _mtx_lock_spin(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line); int _mtx_trylock(struct mtx *m, int opts, const char *file, int line); void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line); @@ -160,16 +159,13 @@ void mtx_unlock_giant(int s); */ #ifndef _get_spin_lock #define _get_spin_lock(mp, tid, opts, file, line) do { \ - critical_t _mtx_crit; \ - _mtx_crit = critical_enter(); \ + critical_enter(); \ if (!_obtain_lock((mp), (tid))) { \ if ((mp)->mtx_lock == (uintptr_t)(tid)) \ (mp)->mtx_recurse++; \ else \ - _mtx_lock_spin((mp), (opts), _mtx_crit, (file), \ - (line)); \ - } else \ - (mp)->mtx_savecrit = _mtx_crit; \ + _mtx_lock_spin((mp), (opts), (file), (line)); \ + } \ } while (0) #endif @@ -189,16 +185,18 @@ void mtx_unlock_giant(int s); * a function call would be too expensive (at least on some architectures). * Since spin locks are not _too_ common, inlining this code is not too big * a deal. + * + * Since we always perform a critical_enter() when attempting to acquire a + * spin lock, we need to always perform a matching critical_exit() when + * releasing a spin lock. This includes the recursion cases. */ #ifndef _rel_spin_lock #define _rel_spin_lock(mp) do { \ - critical_t _mtx_crit = (mp)->mtx_savecrit; \ if (mtx_recursed((mp))) \ (mp)->mtx_recurse--; \ - else { \ + else \ _release_lock_quick((mp)); \ - critical_exit(_mtx_crit); \ - } \ + critical_exit(); \ } while (0) #endif diff --git a/sys/sys/proc.h b/sys/sys/proc.h index beab3160feb..ebf6d36a44f 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -275,6 +275,8 @@ struct thread { struct trapframe *td_frame; /* (k) */ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* Kernel VA of kstack. */ + u_int td_critnest; /* (k) Critical section nest level. */ + critical_t td_savecrit; /* (k) Saved critical section state. */ }; /* diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 3544b1ff95e..70e99bd858f 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -121,6 +121,8 @@ void *phashinit __P((int count, struct malloc_type *type, u_long *nentries)); void cpu_boot __P((int)); void cpu_rootconf __P((void)); +void critical_enter __P((void)); +void critical_exit __P((void)); void init_param1 __P((void)); void init_param2 __P((int physpages)); void tablefull __P((const char *));