180 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_terminate_all()  local
185 	dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch);  in moxart_terminate_all()
187 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_terminate_all()
189 	if (ch->desc) {  in moxart_terminate_all()
190 		moxart_dma_desc_free(&ch->desc->vd);  in moxart_terminate_all()
191 		ch->desc = NULL;  in moxart_terminate_all()
194 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
196 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
198 	vchan_get_all_descriptors(&ch->vc, &head);  in moxart_terminate_all()
199 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_terminate_all()
200 	vchan_dma_desc_free_list(&ch->vc, &head);  in moxart_terminate_all()
208 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_slave_config()  local
211 	ch->cfg = *cfg;  in moxart_slave_config()
213 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_slave_config()
218 	switch (ch->cfg.src_addr_width) {  in moxart_slave_config()
221 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
228 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
235 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
244 	if (ch->cfg.direction == DMA_MEM_TO_DEV) {  in moxart_slave_config()
247 		ctrl |= (ch->line_reqno << 16 &  in moxart_slave_config()
252 		ctrl |= (ch->line_reqno << 24 &  in moxart_slave_config()
256 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_slave_config()
266 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_prep_slave_sg()  local
281 		dev_addr = ch->cfg.src_addr;  in moxart_prep_slave_sg()
282 		dev_width = ch->cfg.src_addr_width;  in moxart_prep_slave_sg()
284 		dev_addr = ch->cfg.dst_addr;  in moxart_prep_slave_sg()
285 		dev_width = ch->cfg.dst_addr_width;  in moxart_prep_slave_sg()
318 	ch->error = 0;  in moxart_prep_slave_sg()
320 	return vchan_tx_prep(&ch->vc, &d->vd, tx_flags);  in moxart_prep_slave_sg()
328 	struct moxart_chan *ch;  in moxart_of_xlate()  local
334 	ch = to_moxart_dma_chan(chan);  in moxart_of_xlate()
335 	ch->line_reqno = dma_spec->args[0];  in moxart_of_xlate()
342 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_alloc_chan_resources()  local
345 		__func__, ch->ch_num);  in moxart_alloc_chan_resources()
346 	ch->allocated = 1;  in moxart_alloc_chan_resources()
353 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_free_chan_resources()  local
355 	vchan_free_chan_resources(&ch->vc);  in moxart_free_chan_resources()
358 		__func__, ch->ch_num);  in moxart_free_chan_resources()
359 	ch->allocated = 0;  in moxart_free_chan_resources()
362 static void moxart_dma_set_params(struct moxart_chan *ch, dma_addr_t src_addr,  in moxart_dma_set_params()  argument
365 	writel(src_addr, ch->base + REG_OFF_ADDRESS_SOURCE);  in moxart_dma_set_params()
366 	writel(dst_addr, ch->base + REG_OFF_ADDRESS_DEST);  in moxart_dma_set_params()
369 static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len)  in moxart_set_transfer_params()  argument
371 	struct moxart_desc *d = ch->desc;  in moxart_set_transfer_params()
380 	writel(d->dma_cycles, ch->base + REG_OFF_CYCLES);  in moxart_set_transfer_params()
382 	dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n",  in moxart_set_transfer_params()
386 static void moxart_start_dma(struct moxart_chan *ch)  in moxart_start_dma()  argument
390 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_start_dma()
392 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_start_dma()
395 static void moxart_dma_start_sg(struct moxart_chan *ch, unsigned int idx)  in moxart_dma_start_sg()  argument
397 	struct moxart_desc *d = ch->desc;  in moxart_dma_start_sg()
398 	struct moxart_sg *sg = ch->desc->sg + idx;  in moxart_dma_start_sg()
400 	if (ch->desc->dma_dir == DMA_MEM_TO_DEV)  in moxart_dma_start_sg()
401 		moxart_dma_set_params(ch, sg->addr, d->dev_addr);  in moxart_dma_start_sg()
402 	else if (ch->desc->dma_dir == DMA_DEV_TO_MEM)  in moxart_dma_start_sg()
403 		moxart_dma_set_params(ch, d->dev_addr, sg->addr);  in moxart_dma_start_sg()
405 	moxart_set_transfer_params(ch, sg->len);  in moxart_dma_start_sg()
407 	moxart_start_dma(ch);  in moxart_dma_start_sg()
412 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_dma_start_desc()  local
415 	vd = vchan_next_desc(&ch->vc);  in moxart_dma_start_desc()
418 		ch->desc = NULL;  in moxart_dma_start_desc()
424 	ch->desc = to_moxart_dma_desc(&vd->tx);  in moxart_dma_start_desc()
425 	ch->sgidx = 0;  in moxart_dma_start_desc()
427 	moxart_dma_start_sg(ch, 0);  in moxart_dma_start_desc()
432 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_issue_pending()  local
435 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_issue_pending()
436 	if (vchan_issue_pending(&ch->vc) && !ch->desc)  in moxart_issue_pending()
438 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_issue_pending()
453 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch)  in moxart_dma_desc_size_in_flight()  argument
458 	size = moxart_dma_desc_size(ch->desc, ch->sgidx);  in moxart_dma_desc_size_in_flight()
459 	cycles = readl(ch->base + REG_OFF_CYCLES);  in moxart_dma_desc_size_in_flight()
460 	completed_cycles = (ch->desc->dma_cycles - cycles);  in moxart_dma_desc_size_in_flight()
461 	size -= completed_cycles << es_bytes[ch->desc->es];  in moxart_dma_desc_size_in_flight()
463 	dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size);  in moxart_dma_desc_size_in_flight()
472 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_tx_status()  local
483 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_tx_status()
484 	vd = vchan_find_desc(&ch->vc, cookie);  in moxart_tx_status()
488 	} else if (ch->desc && ch->desc->vd.tx.cookie == cookie) {  in moxart_tx_status()
489 		txstate->residue = moxart_dma_desc_size_in_flight(ch);  in moxart_tx_status()
491 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_tx_status()
493 	if (ch->error)  in moxart_tx_status()
516 	struct moxart_chan *ch = &mc->slave_chans[0];  in moxart_dma_interrupt()  local
520 	dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);  in moxart_dma_interrupt()
522 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_dma_interrupt()
523 		if (!ch->allocated)  in moxart_dma_interrupt()
526 		ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
528 		dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n",  in moxart_dma_interrupt()
529 			__func__, ch, ch->base, ctrl);  in moxart_dma_interrupt()
533 			if (ch->desc) {  in moxart_dma_interrupt()
534 				spin_lock(&ch->vc.lock);  in moxart_dma_interrupt()
535 				if (++ch->sgidx < ch->desc->sglen) {  in moxart_dma_interrupt()
536 					moxart_dma_start_sg(ch, ch->sgidx);  in moxart_dma_interrupt()
538 					vchan_cookie_complete(&ch->desc->vd);  in moxart_dma_interrupt()
539 					moxart_dma_start_desc(&ch->vc.chan);  in moxart_dma_interrupt()
541 				spin_unlock(&ch->vc.lock);  in moxart_dma_interrupt()
547 			ch->error = 1;  in moxart_dma_interrupt()
550 		writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
563 	struct moxart_chan *ch;  in moxart_probe()  local
586 	ch = &mdc->slave_chans[0];  in moxart_probe()
587 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_probe()
588 		ch->ch_num = i;  in moxart_probe()
589 		ch->base = dma_base_addr + i * REG_OFF_CHAN_SIZE;  in moxart_probe()
590 		ch->allocated = 0;  in moxart_probe()
592 		ch->vc.desc_free = moxart_dma_desc_free;  in moxart_probe()
593 		vchan_init(&ch->vc, &mdc->dma_slave);  in moxart_probe()
595 		dev_dbg(dev, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n",  in moxart_probe()
596 			__func__, i, ch->ch_num, ch->base);  in moxart_probe()