87 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)  in arch_try_cmpxchg_niai8()  argument
93 		"	cs	%[old],%[new],%[lock]\n"  in arch_try_cmpxchg_niai8()
94 		: [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc)  in arch_try_cmpxchg_niai8()
102 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)  in arch_try_cmpxchg_niai8()  argument
104 	int expected = old;  in arch_try_cmpxchg_niai8()
108 		"	cs	%[old],%[new],%[lock]\n"  in arch_try_cmpxchg_niai8()
109 		: [old] "+d" (old), [lock] "+Q" (*lock)  in arch_try_cmpxchg_niai8()
112 	return expected == old;  in arch_try_cmpxchg_niai8()
140 	int lockval, ix, node_id, tail_id, old, new, owner, count;  in arch_spin_lock_queued()  local
150 	old = READ_ONCE(lp->lock);  in arch_spin_lock_queued()
152 		if ((old & _Q_LOCK_CPU_MASK) == 0 &&  in arch_spin_lock_queued()
153 		    (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {  in arch_spin_lock_queued()
161 			new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;  in arch_spin_lock_queued()
162 			if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
169 		new = node_id | (old & _Q_LOCK_MASK);  in arch_spin_lock_queued()
170 		if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
174 	tail_id = old & _Q_TAIL_MASK;  in arch_spin_lock_queued()
181 	owner = arch_spin_yield_target(old, node);  in arch_spin_lock_queued()
193 			owner = arch_spin_yield_target(old, node);  in arch_spin_lock_queued()
202 		old = READ_ONCE(lp->lock);  in arch_spin_lock_queued()
203 		owner = old & _Q_LOCK_CPU_MASK;  in arch_spin_lock_queued()
205 			tail_id = old & _Q_TAIL_MASK;  in arch_spin_lock_queued()
207 			if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
233 	int lockval, old, new, owner, count;  in arch_spin_lock_classic()  local
244 		old = arch_load_niai4(&lp->lock);  in arch_spin_lock_classic()
245 		owner = old & _Q_LOCK_CPU_MASK;  in arch_spin_lock_classic()
248 			new = (old & _Q_TAIL_MASK) | lockval;  in arch_spin_lock_classic()
249 			if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) {  in arch_spin_lock_classic()
312 	int old;  in arch_write_lock_wait()  local
321 		old = READ_ONCE(rw->cnts);  in arch_write_lock_wait()
322 		if ((old & 0x1ffff) == 0 &&  in arch_write_lock_wait()
323 		    arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000))  in arch_write_lock_wait()