1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
3  * Copyright(c) 2015 - 2018 Intel Corporation.
11 #include <linux/wait.h>
17  * typedef (*restart_t)() - restart callback
57  * @list: used to add/insert into QP/PQ wait lists
63  * @lock: lock protected head of wait queue
65  * @wait_dma: wait for sdma_busy == 0
66  * @wait_pio: wait for pio_busy == 0
71  * @flags: wait flags (one per QP)
72  * @wait: SE array for multiple legs
102 		struct iowait_work *wait,
107 	void (*wakeup)(struct iowait *wait, int reason);
108 	void (*sdma_drained)(struct iowait *wait);
109 	void (*init_priority)(struct iowait *wait);
121 	struct iowait_work wait[IOWAIT_SES];  member
126 void iowait_set_flag(struct iowait *wait, u32 flag);
127 bool iowait_flag_set(struct iowait *wait, u32 flag);
128 void iowait_clear_flag(struct iowait *wait, u32 flag);
130 void iowait_init(struct iowait *wait, u32 tx_limit,
134 			      struct iowait_work *wait,
138 		 void (*wakeup)(struct iowait *wait, int reason),
139 		 void (*sdma_drained)(struct iowait *wait),
140 		 void (*init_priority)(struct iowait *wait));
143  * iowait_schedule() - schedule the default send engine work
144  * @wait: wait struct to schedule
148 static inline bool iowait_schedule(struct iowait *wait,  in iowait_schedule()  argument
151 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);  in iowait_schedule()
155  * iowait_tid_schedule - schedule the tid SE
156  * @wait: the iowait structure
157  * @wq: the work queue
160 static inline bool iowait_tid_schedule(struct iowait *wait,  in iowait_tid_schedule()  argument
163 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);  in iowait_tid_schedule()
167  * iowait_sdma_drain() - wait for DMAs to drain
169  * @wait: iowait structure
174 static inline void iowait_sdma_drain(struct iowait *wait)  in iowait_sdma_drain()  argument
176 	wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));  in iowait_sdma_drain()
180  * iowait_sdma_pending() - return sdma pending count
182  * @wait: iowait structure
185 static inline int iowait_sdma_pending(struct iowait *wait)  in iowait_sdma_pending()  argument
187 	return atomic_read(&wait->sdma_busy);  in iowait_sdma_pending()
191  * iowait_sdma_inc - note sdma io pending
192  * @wait: iowait structure
194 static inline void iowait_sdma_inc(struct iowait *wait)  in iowait_sdma_inc()  argument
196 	atomic_inc(&wait->sdma_busy);  in iowait_sdma_inc()
200  * iowait_sdma_add - add count to pending
201  * @wait: iowait structure
203 static inline void iowait_sdma_add(struct iowait *wait, int count)  in iowait_sdma_add()  argument
205 	atomic_add(count, &wait->sdma_busy);  in iowait_sdma_add()
209  * iowait_sdma_dec - note sdma complete
210  * @wait: iowait structure
212 static inline int iowait_sdma_dec(struct iowait *wait)  in iowait_sdma_dec()  argument
214 	if (!wait)  in iowait_sdma_dec()
216 	return atomic_dec_and_test(&wait->sdma_busy);  in iowait_sdma_dec()
220  * iowait_pio_drain() - wait for pios to drain
222  * @wait: iowait structure
227 static inline void iowait_pio_drain(struct iowait *wait)  in iowait_pio_drain()  argument
229 	wait_event_timeout(wait->wait_pio,  in iowait_pio_drain()
230 			   !atomic_read(&wait->pio_busy),  in iowait_pio_drain()
235  * iowait_pio_pending() - return pio pending count
237  * @wait: iowait structure
240 static inline int iowait_pio_pending(struct iowait *wait)  in iowait_pio_pending()  argument
242 	return atomic_read(&wait->pio_busy);  in iowait_pio_pending()
246  * iowait_pio_inc - note pio pending
247  * @wait: iowait structure
249 static inline void iowait_pio_inc(struct iowait *wait)  in iowait_pio_inc()  argument
251 	atomic_inc(&wait->pio_busy);  in iowait_pio_inc()
255  * iowait_pio_dec - note pio complete
256  * @wait: iowait structure
258 static inline int iowait_pio_dec(struct iowait *wait)  in iowait_pio_dec()  argument
260 	if (!wait)  in iowait_pio_dec()
262 	return atomic_dec_and_test(&wait->pio_busy);  in iowait_pio_dec()
266  * iowait_drain_wakeup() - trigger iowait_drain() waiter
268  * @wait: iowait structure
272 static inline void iowait_drain_wakeup(struct iowait *wait)  in iowait_drain_wakeup()  argument
274 	wake_up(&wait->wait_dma);  in iowait_drain_wakeup()
275 	wake_up(&wait->wait_pio);  in iowait_drain_wakeup()
276 	if (wait->sdma_drained)  in iowait_drain_wakeup()
277 		wait->sdma_drained(wait);  in iowait_drain_wakeup()
281  * iowait_get_txhead() - get packet off of iowait list
283  * @wait: iowait_work structure
285 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)  in iowait_get_txhead()  argument
289 	if (!list_empty(&wait->tx_head)) {  in iowait_get_txhead()
291 			&wait->tx_head,  in iowait_get_txhead()
294 		list_del_init(&tx->list);  in iowait_get_txhead()
304 	if (!list_empty(&w->tx_head)) {  in iowait_get_desc()
305 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,  in iowait_get_desc()
307 		num_desc = tx->num_desc;  in iowait_get_desc()
308 		if (tx->flags & SDMA_TXREQ_F_VIP)  in iowait_get_desc()
309 			w->iow->priority++;  in iowait_get_desc()
318 	num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);  in iowait_get_all_desc()
319 	num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);  in iowait_get_all_desc()
327 	if (!list_empty(&w->tx_head)) {  in iowait_update_priority()
328 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,  in iowait_update_priority()
330 		if (tx->flags & SDMA_TXREQ_F_VIP)  in iowait_update_priority()
331 			w->iow->priority++;  in iowait_update_priority()
337 	iowait_update_priority(&w->wait[IOWAIT_IB_SE]);  in iowait_update_all_priority()
338 	iowait_update_priority(&w->wait[IOWAIT_TID_SE]);  in iowait_update_all_priority()
343 	w->priority = 0;  in iowait_init_priority()
344 	if (w->init_priority)  in iowait_init_priority()
345 		w->init_priority(w);  in iowait_init_priority()
355  * iowait_queue - Put the iowait on a wait queue
358  * @wait_head: the wait queue
361  * wait queue after a resource (eg, sdma descriptor or pio
368 	 * To play fair, insert the iowait at the tail of the wait queue if it  in iowait_queue()
374 		w->starved_cnt = 0;  in iowait_queue()
376 		w->starved_cnt++;  in iowait_queue()
378 	if (w->priority > 0 || !pkts_sent)  in iowait_queue()
379 		list_add(&w->list, wait_head);  in iowait_queue()
381 		list_add_tail(&w->list, wait_head);  in iowait_queue()
385  * iowait_starve_clear - clear the wait queue's starve count
395 		w->starved_cnt = 0;  in iowait_starve_clear()
404  * iowait_packet_queued() - determine if a packet is queued
405  * @wait: the iowait_work structure
407 static inline bool iowait_packet_queued(struct iowait_work *wait)  in iowait_packet_queued()  argument
409 	return !list_empty(&wait->tx_head);  in iowait_packet_queued()
413  * inc_wait_count - increment wait counts
421 	w->iow->tx_count++;  in iowait_inc_wait_count()
422 	w->iow->count += n;  in iowait_inc_wait_count()
426  * iowait_get_tid_work - return iowait_work for tid SE
431 	return &w->wait[IOWAIT_TID_SE];  in iowait_get_tid_work()
435  * iowait_get_ib_work - return iowait_work for ib SE
440 	return &w->wait[IOWAIT_IB_SE];  in iowait_get_ib_work()
444  * iowait_ioww_to_iow - return iowait given iowait_work
450 		return w->iow;  in iowait_ioww_to_iow()