189 	 * for split ring, it just contains last used index
200 		/* Available for split ring */
201 		struct vring_virtqueue_split split;  member
441  * Split ring specific functions - *_split().
567 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
579 		desc = vq->split.vring.desc;  in virtqueue_add_split()
580 		extra = vq->split.desc_extra;  in virtqueue_add_split()
637 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
648 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
649 					 vq->split.desc_extra,  in virtqueue_add_split()
660 		vq->free_head = vq->split.desc_extra[head].next;  in virtqueue_add_split()
665 	vq->split.desc_state[head].data = data;  in virtqueue_add_split()
667 		vq->split.desc_state[head].indir_desc = desc;  in virtqueue_add_split()
669 		vq->split.desc_state[head].indir_desc = ctx;  in virtqueue_add_split()
673 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
674 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
679 	vq->split.avail_idx_shadow++;  in virtqueue_add_split()
680 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
681 						vq->split.avail_idx_shadow);  in virtqueue_add_split()
727 	old = vq->split.avail_idx_shadow - vq->num_added;  in virtqueue_kick_prepare_split()
728 	new = vq->split.avail_idx_shadow;  in virtqueue_kick_prepare_split()
736 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
739 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
755 	vq->split.desc_state[head].data = NULL;  in detach_buf_split()
757 	extra = vq->split.desc_extra;  in detach_buf_split()
762 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
764 		i = vq->split.desc_extra[i].next;  in detach_buf_split()
769 	vq->split.desc_extra[i].next = vq->free_head;  in detach_buf_split()
777 				vq->split.desc_state[head].indir_desc;  in detach_buf_split()
783 		len = vq->split.desc_extra[head].len;  in detach_buf_split()
785 		BUG_ON(!(vq->split.desc_extra[head].flags &  in detach_buf_split()
799 		vq->split.desc_state[head].indir_desc = NULL;  in detach_buf_split()
801 		*ctx = vq->split.desc_state[head].indir_desc;  in detach_buf_split()
808 			vq->split.vring.used->idx);  in more_used_split()
836 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
838 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
840 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
842 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
846 	if (unlikely(!vq->split.desc_state[i].data)) {  in virtqueue_get_buf_ctx_split()
852 	ret = vq->split.desc_state[i].data;  in virtqueue_get_buf_ctx_split()
858 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))  in virtqueue_get_buf_ctx_split()
860 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
873 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {  in virtqueue_disable_cb_split()
874 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_disable_cb_split()
885 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
887 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
889 						vq->split.avail_flags_shadow);  in virtqueue_disable_cb_split()
905 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_prepare_split()
906 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_prepare_split()
908 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
910 						vq->split.avail_flags_shadow);  in virtqueue_enable_cb_prepare_split()
912 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
923 			vq->split.vring.used->idx);  in virtqueue_poll_split()
938 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_delayed_split()
939 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_delayed_split()
941 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
943 						vq->split.avail_flags_shadow);  in virtqueue_enable_cb_delayed_split()
946 	bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;  in virtqueue_enable_cb_delayed_split()
949 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
952 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
970 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
971 		if (!vq->split.desc_state[i].data)  in virtqueue_detach_unused_buf_split()
974 		buf = vq->split.desc_state[i].data;  in virtqueue_detach_unused_buf_split()
976 		vq->split.avail_idx_shadow--;  in virtqueue_detach_unused_buf_split()
977 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
978 				vq->split.avail_idx_shadow);  in virtqueue_detach_unused_buf_split()
983 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
1012 	num = vq->split.vring.num;  in virtqueue_reinit_split()
1014 	vq->split.vring.avail->flags = 0;  in virtqueue_reinit_split()
1015 	vq->split.vring.avail->idx = 0;  in virtqueue_reinit_split()
1018 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reinit_split()
1020 	vq->split.vring.used->flags = 0;  in virtqueue_reinit_split()
1021 	vq->split.vring.used->idx = 0;  in virtqueue_reinit_split()
1024 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reinit_split()
1028 	virtqueue_vring_init_split(&vq->split, vq);  in virtqueue_reinit_split()
1034 	vq->split = *vring_split;  in virtqueue_vring_attach_split()
1229 				      vq->split.vring_align,  in virtqueue_resize_split()
1230 				      vq->split.may_reduce_num,  in virtqueue_resize_split()
2455  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2475  * virtqueue_notify - second half of split virtqueue_kick call.
2808 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2922 					 vq->split.queue_size_in_bytes,  in vring_free()
2923 					 vq->split.vring.desc,  in vring_free()
2924 					 vq->split.queue_dma_addr,  in vring_free()
2929 		kfree(vq->split.desc_state);  in vring_free()
2930 		kfree(vq->split.desc_extra);  in vring_free()
2959 		next = vq->split.avail_idx_shadow;  in vring_notification_data()
3006 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
3092 	return vq->split.queue_dma_addr;  in virtqueue_get_desc_addr()
3105 	return vq->split.queue_dma_addr +  in virtqueue_get_avail_addr()
3106 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
3119 	return vq->split.queue_dma_addr +  in virtqueue_get_used_addr()
3120 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
3124 /* Only available for split ring */
3127 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()