From 718c2b5b1164652eb8aab144f5c3493d2e4c160b Mon Sep 17 00:00:00 2001
From: Marius Strobl <marius@FreeBSD.org>
Date: Wed, 23 Jan 2013 22:52:20 +0000
Subject: [PATCH] Revert the part of r239864 which removed obtaining the SMP
 mutex around reading registers from other CPUs. As it turns out, the hardware
 doesn't really like concurrent IPI'ing causing adverse effects. Also the
 thought deadlock when using this spin lock here and the targeted CPU(s) are
 also holding or in case of nested locks can't actually happen. This is due to
 the fact that on sparc64, spinlock_enter() only raises the PIL but doesn't
 disable interrupts completely. Thus direct cross calls as used for the
 register reading (and all other MD IPI needs) still will be executed by the
 targeted CPU(s) in that case.

MFC after:	3 days
---
 sys/sparc64/include/smp.h  | 20 ++------------------
 sys/sparc64/sparc64/tick.c |  4 ++--
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index db173095e64..2e1110b95c8 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -42,6 +42,7 @@
 #include <sys/cpuset.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
+#include <sys/smp.h>
 
 #include <machine/intr_machdep.h>
 #include <machine/pcb.h>
@@ -202,6 +203,7 @@ ipi_rd(u_int cpu, void *func, u_long *val)
 		return (NULL);
 	sched_pin();
 	ira = &ipi_rd_args;
+	mtx_lock_spin(&smp_ipi_mtx);
 	CPU_SETOF(cpu, &ira->ira_mask);
 	ira->ira_val = val;
 	cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira);
@@ -298,18 +300,6 @@ ipi_wait(void *cookie)
 	}
 }
 
-static __inline void
-ipi_wait_unlocked(void *cookie)
-{
-	volatile cpuset_t *mask;
-
-	if ((mask = cookie) != NULL) {
-		while (!CPU_EMPTY(mask))
-			;
-		sched_unpin();
-	}
-}
-
 #endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */
 
 #endif /* !LOCORE */
@@ -367,12 +357,6 @@ ipi_wait(void *cookie __unused)
 
 }
 
-static __inline void
-ipi_wait_unlocked(void *cookie __unused)
-{
-
-}
-
 static __inline void
 tl_ipi_cheetah_dcache_page_inval(void)
 {
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 69062ecb818..cfd5776e7c9 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -334,7 +334,7 @@ stick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		stick = rdstick();
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_stick_rd, &stick));
+		ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
 	sched_unpin();
 	return (stick);
 }
@@ -348,7 +348,7 @@ tick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		tick = rd(tick);
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_tick_rd, &tick));
+		ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
 	sched_unpin();
 	return (tick);
 }