--- //depot/projects/smpng/sys/amd64/include/atomic.h	2005/09/15 19:40:43
+++ //depot/user/jhb/ktrace/amd64/include/atomic.h	2005/09/15 20:07:23
@@ -74,7 +74,7 @@
 int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src);
 int atomic_cmpset_long(volatile u_long *dst, u_long exp, u_long src);
 
-#define	ATOMIC_STORE_LOAD(TYPE, LOP, SOP)			\
+#define	ATOMIC_STORE_LOAD(TYPE)					\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p);	\
 void		atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 
@@ -157,13 +157,12 @@
 #if defined(_KERNEL) && !defined(SMP)
 
 /*
- * We assume that a = b will do atomic loads and stores.  However, on a
- * PentiumPro or higher, reads may pass writes, so for that case we have
- * to use a serializing instruction (i.e. with LOCK) to do the load in
- * SMP kernels.  For UP kernels, however, the cache of the single processor
+ * We assume that a = b will do atomic loads and stores.  However, reads
+ * may pass writes, so we have to use fences in SMP kernels to preserve
+ * ordering.  For UP kernels, however, the cache of the single processor
  * is always consistent, so we don't need any memory barriers.
  */
-#define	ATOMIC_STORE_LOAD(TYPE, LOP, SOP)		\
+#define	ATOMIC_STORE_LOAD(TYPE)				\
 static __inline u_##TYPE				\
 atomic_load_acq_##TYPE(volatile u_##TYPE *p)		\
 {							\
@@ -179,35 +178,26 @@
 
 #else /* defined(SMP) */
 
-#define	ATOMIC_STORE_LOAD(TYPE, LOP, SOP)		\
+#define	ATOMIC_STORE_LOAD(TYPE)				\
 static __inline u_##TYPE				\
 atomic_load_acq_##TYPE(volatile u_##TYPE *p)		\
 {							\
-	u_##TYPE res;					\
+	u_##TYPE v;					\
 							\
-	__asm __volatile(__XSTRING(MPLOCKED) LOP	\
-	: "=a" (res),			/* 0 (result) */\
-	  "=m" (*p)			/* 1 */		\
-	: "m" (*p)			/* 2 */		\
-	: "memory");				 	\
-							\
-	return (res);					\
+	v = *p;						\
+	__asm __volatile("lfence" ::: "memory");	\
+	return (v);					\
 }							\
 							\
-/*							\
- * The XCHG instruction asserts LOCK automagically.	\
- */							\
 static __inline void					\
 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 {							\
-	__asm __volatile(SOP				\
-	: "=m" (*p),			/* 0 */		\
-	  "+r" (v)			/* 1 */		\
-	: "m" (*p));			/* 2 */		\
+	__asm __volatile("sfence");			\
+	*p = v;						\
 }							\
 struct __hack
 
-#endif /* SMP */
+#endif	/* !defined(SMP) */
 
 #endif /* KLD_MODULE || !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
 
@@ -231,10 +221,10 @@
 ATOMIC_ASM(add,	     long,  "addq %1,%0",  "ir",  v);
 ATOMIC_ASM(subtract, long,  "subq %1,%0",  "ir",  v);
 
-ATOMIC_STORE_LOAD(char,	"cmpxchgb %b0,%1", "xchgb %b1,%0");
-ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0");
-ATOMIC_STORE_LOAD(int,	"cmpxchgl %0,%1",  "xchgl %1,%0");
-ATOMIC_STORE_LOAD(long,	"cmpxchgq %0,%1",  "xchgq %1,%0");
+ATOMIC_STORE_LOAD(char);
+ATOMIC_STORE_LOAD(short);
+ATOMIC_STORE_LOAD(int);
+ATOMIC_STORE_LOAD(long);
 
 #undef ATOMIC_ASM
 #undef ATOMIC_STORE_LOAD
--- //depot/projects/smpng/sys/amd64/include/bus.h	2005/06/17 19:55:36
+++ //depot/user/jhb/ktrace/amd64/include/bus.h	2005/07/18 18:15:59
@@ -997,10 +985,13 @@
 		  bus_size_t offset __unused, bus_size_t len __unused, int flags)
 {
 #ifdef __GNUCLIKE_ASM
-	if (flags & BUS_SPACE_BARRIER_READ)
-		__asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory");
+	if ((flags & (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) ==
+	    (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE))
+		__asm __volatile("mfence" : : : "memory");
+	else if (flags & BUS_SPACE_BARRIER_READ)
+		__asm __volatile("lfence" : : : "memory");
 	else
-		__asm __volatile("" : : : "memory");
+		__asm __volatile("sfence");
 #endif
 }