1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) VF Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/pci.h>
9 #include <linux/etherdevice.h>
10 #include <linux/vmalloc.h>
11
12 #include "octep_vf_config.h"
13 #include "octep_vf_main.h"
14
octep_vf_oq_reset_indices(struct octep_vf_oq * oq)15 static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq)
16 {
17 oq->host_read_idx = 0;
18 oq->host_refill_idx = 0;
19 oq->refill_count = 0;
20 oq->last_pkt_count = 0;
21 oq->pkts_pending = 0;
22 }
23
24 /**
25 * octep_vf_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
26 *
27 * @oq: Octeon Rx queue data structure.
28 *
29 * Return: 0, if successfully filled receive buffers for all descriptors.
30 * -ENOMEM, if failed to allocate a buffer or failed to map for DMA.
31 */
octep_vf_oq_fill_ring_buffers(struct octep_vf_oq * oq)32 static int octep_vf_oq_fill_ring_buffers(struct octep_vf_oq *oq)
33 {
34 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
35 struct page *page;
36 u32 i;
37
38 for (i = 0; i < oq->max_count; i++) {
39 page = dev_alloc_page();
40 if (unlikely(!page)) {
41 dev_err(oq->dev, "Rx buffer alloc failed\n");
42 goto rx_buf_alloc_err;
43 }
44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
45 PAGE_SIZE,
46 DMA_FROM_DEVICE);
47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
48 dev_err(oq->dev,
49 "OQ-%d buffer alloc: DMA mapping error!\n",
50 oq->q_no);
51 goto dma_map_err;
52 }
53 oq->buff_info[i].page = page;
54 }
55
56 return 0;
57
58 dma_map_err:
59 put_page(page);
60 rx_buf_alloc_err:
61 while (i) {
62 i--;
63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
64 put_page(oq->buff_info[i].page);
65 oq->buff_info[i].page = NULL;
66 }
67
68 return -ENOMEM;
69 }
70
71 /**
72 * octep_vf_oq_refill() - refill buffers for used Rx ring descriptors.
73 *
74 * @oct: Octeon device private data structure.
75 * @oq: Octeon Rx queue data structure.
76 *
77 * Return: number of descriptors successfully refilled with receive buffers.
78 */
octep_vf_oq_refill(struct octep_vf_device * oct,struct octep_vf_oq * oq)79 static int octep_vf_oq_refill(struct octep_vf_device *oct, struct octep_vf_oq *oq)
80 {
81 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
82 struct page *page;
83 u32 refill_idx, i;
84
85 refill_idx = oq->host_refill_idx;
86 for (i = 0; i < oq->refill_count; i++) {
87 page = dev_alloc_page();
88 if (unlikely(!page)) {
89 dev_err(oq->dev, "refill: rx buffer alloc failed\n");
90 oq->stats->alloc_failures++;
91 break;
92 }
93
94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
95 PAGE_SIZE, DMA_FROM_DEVICE);
96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
97 dev_err(oq->dev,
98 "OQ-%d buffer refill: DMA mapping error!\n",
99 oq->q_no);
100 put_page(page);
101 oq->stats->alloc_failures++;
102 break;
103 }
104 oq->buff_info[refill_idx].page = page;
105 refill_idx++;
106 if (refill_idx == oq->max_count)
107 refill_idx = 0;
108 }
109 oq->host_refill_idx = refill_idx;
110 oq->refill_count -= i;
111
112 return i;
113 }
114
115 /**
116 * octep_vf_setup_oq() - Setup a Rx queue.
117 *
118 * @oct: Octeon device private data structure.
119 * @q_no: Rx queue number to be setup.
120 *
121 * Allocate resources for a Rx queue.
122 */
octep_vf_setup_oq(struct octep_vf_device * oct,int q_no)123 static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no)
124 {
125 struct octep_vf_oq *oq;
126 u32 desc_ring_size;
127
128 oq = vzalloc(sizeof(*oq));
129 if (!oq)
130 goto create_oq_fail;
131 oct->oq[q_no] = oq;
132
133 oq->octep_vf_dev = oct;
134 oq->netdev = oct->netdev;
135 oq->dev = &oct->pdev->dev;
136 oq->q_no = q_no;
137 oq->stats = &oct->stats_oq[q_no];
138 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
139 oq->ring_size_mask = oq->max_count - 1;
140 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
141 oq->max_single_buffer_size = oq->buffer_size - OCTEP_VF_OQ_RESP_HW_SIZE;
142
143 /* When the hardware/firmware supports additional capabilities,
144 * additional header is filled-in by Octeon after length field in
145 * Rx packets. this header contains additional packet information.
146 */
147 if (oct->fw_info.rx_ol_flags)
148 oq->max_single_buffer_size -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
149
150 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
151
152 desc_ring_size = oq->max_count * OCTEP_VF_OQ_DESC_SIZE;
153 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
154 &oq->desc_ring_dma, GFP_KERNEL);
155
156 if (unlikely(!oq->desc_ring)) {
157 dev_err(oq->dev,
158 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);
159 goto desc_dma_alloc_err;
160 }
161
162 oq->buff_info = vzalloc(oq->max_count * OCTEP_VF_OQ_RECVBUF_SIZE);
163
164 if (unlikely(!oq->buff_info)) {
165 dev_err(&oct->pdev->dev,
166 "Failed to allocate buffer info for OQ-%d\n", q_no);
167 goto buf_list_err;
168 }
169
170 if (octep_vf_oq_fill_ring_buffers(oq))
171 goto oq_fill_buff_err;
172
173 octep_vf_oq_reset_indices(oq);
174 oct->hw_ops.setup_oq_regs(oct, q_no);
175 oct->num_oqs++;
176
177 return 0;
178
179 oq_fill_buff_err:
180 vfree(oq->buff_info);
181 oq->buff_info = NULL;
182 buf_list_err:
183 dma_free_coherent(oq->dev, desc_ring_size,
184 oq->desc_ring, oq->desc_ring_dma);
185 oq->desc_ring = NULL;
186 desc_dma_alloc_err:
187 vfree(oq);
188 oct->oq[q_no] = NULL;
189 create_oq_fail:
190 return -ENOMEM;
191 }
192
193 /**
194 * octep_vf_oq_free_ring_buffers() - Free ring buffers.
195 *
196 * @oq: Octeon Rx queue data structure.
197 *
198 * Free receive buffers in unused Rx queue descriptors.
199 */
octep_vf_oq_free_ring_buffers(struct octep_vf_oq * oq)200 static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq)
201 {
202 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
203 int i;
204
205 if (!oq->desc_ring || !oq->buff_info)
206 return;
207
208 for (i = 0; i < oq->max_count; i++) {
209 if (oq->buff_info[i].page) {
210 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
211 PAGE_SIZE, DMA_FROM_DEVICE);
212 put_page(oq->buff_info[i].page);
213 oq->buff_info[i].page = NULL;
214 desc_ring[i].buffer_ptr = 0;
215 }
216 }
217 octep_vf_oq_reset_indices(oq);
218 }
219
220 /**
221 * octep_vf_free_oq() - Free Rx queue resources.
222 *
223 * @oq: Octeon Rx queue data structure.
224 *
225 * Free all resources of a Rx queue.
226 */
octep_vf_free_oq(struct octep_vf_oq * oq)227 static int octep_vf_free_oq(struct octep_vf_oq *oq)
228 {
229 struct octep_vf_device *oct = oq->octep_vf_dev;
230 int q_no = oq->q_no;
231
232 octep_vf_oq_free_ring_buffers(oq);
233
234 vfree(oq->buff_info);
235
236 if (oq->desc_ring)
237 dma_free_coherent(oq->dev,
238 oq->max_count * OCTEP_VF_OQ_DESC_SIZE,
239 oq->desc_ring, oq->desc_ring_dma);
240
241 vfree(oq);
242 oct->oq[q_no] = NULL;
243 oct->num_oqs--;
244 return 0;
245 }
246
247 /**
248 * octep_vf_setup_oqs() - setup resources for all Rx queues.
249 *
250 * @oct: Octeon device private data structure.
251 */
octep_vf_setup_oqs(struct octep_vf_device * oct)252 int octep_vf_setup_oqs(struct octep_vf_device *oct)
253 {
254 int i, retval = 0;
255
256 oct->num_oqs = 0;
257 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
258 retval = octep_vf_setup_oq(oct, i);
259 if (retval) {
260 dev_err(&oct->pdev->dev,
261 "Failed to setup OQ(RxQ)-%d.\n", i);
262 goto oq_setup_err;
263 }
264 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
265 }
266
267 return 0;
268
269 oq_setup_err:
270 while (i) {
271 i--;
272 octep_vf_free_oq(oct->oq[i]);
273 }
274 return retval;
275 }
276
277 /**
278 * octep_vf_oq_dbell_init() - Initialize Rx queue doorbell.
279 *
280 * @oct: Octeon device private data structure.
281 *
282 * Write number of descriptors to Rx queue doorbell register.
283 */
octep_vf_oq_dbell_init(struct octep_vf_device * oct)284 void octep_vf_oq_dbell_init(struct octep_vf_device *oct)
285 {
286 int i;
287
288 for (i = 0; i < oct->num_oqs; i++)
289 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
290 }
291
292 /**
293 * octep_vf_free_oqs() - Free resources of all Rx queues.
294 *
295 * @oct: Octeon device private data structure.
296 */
octep_vf_free_oqs(struct octep_vf_device * oct)297 void octep_vf_free_oqs(struct octep_vf_device *oct)
298 {
299 int i;
300
301 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
302 if (!oct->oq[i])
303 continue;
304 octep_vf_free_oq(oct->oq[i]);
305 dev_dbg(&oct->pdev->dev,
306 "Successfully freed OQ(RxQ)-%d.\n", i);
307 }
308 }
309
310 /**
311 * octep_vf_oq_check_hw_for_pkts() - Check for new Rx packets.
312 *
313 * @oct: Octeon device private data structure.
314 * @oq: Octeon Rx queue data structure.
315 *
316 * Return: packets received after previous check.
317 */
octep_vf_oq_check_hw_for_pkts(struct octep_vf_device * oct,struct octep_vf_oq * oq)318 static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct,
319 struct octep_vf_oq *oq)
320 {
321 u32 pkt_count, new_pkts;
322
323 pkt_count = readl(oq->pkts_sent_reg);
324 new_pkts = pkt_count - oq->last_pkt_count;
325
326 /* Clear the hardware packets counter register if the rx queue is
327 * being processed continuously with-in a single interrupt and
328 * reached half its max value.
329 * this counter is not cleared every time read, to save write cycles.
330 */
331 if (unlikely(pkt_count > 0xF0000000U)) {
332 writel(pkt_count, oq->pkts_sent_reg);
333 pkt_count = readl(oq->pkts_sent_reg);
334 new_pkts += pkt_count;
335 }
336 oq->last_pkt_count = pkt_count;
337 oq->pkts_pending += new_pkts;
338 return new_pkts;
339 }
340
341 /**
342 * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack.
343 *
344 * @oct: Octeon device private data structure.
345 * @oq: Octeon Rx queue data structure.
346 * @pkts_to_process: number of packets to be processed.
347 *
348 * Process the new packets in Rx queue.
349 * Packets larger than single Rx buffer arrive in consecutive descriptors.
350 * But, count returned by the API only accounts full packets, not fragments.
351 *
352 * Return: number of packets processed and pushed to stack.
353 */
__octep_vf_oq_process_rx(struct octep_vf_device * oct,struct octep_vf_oq * oq,u16 pkts_to_process)354 static int __octep_vf_oq_process_rx(struct octep_vf_device *oct,
355 struct octep_vf_oq *oq, u16 pkts_to_process)
356 {
357 struct octep_vf_oq_resp_hw_ext *resp_hw_ext = NULL;
358 netdev_features_t feat = oq->netdev->features;
359 struct octep_vf_rx_buffer *buff_info;
360 struct octep_vf_oq_resp_hw *resp_hw;
361 u32 pkt, rx_bytes, desc_used;
362 u16 data_offset, rx_ol_flags;
363 struct sk_buff *skb;
364 u32 read_idx;
365
366 read_idx = oq->host_read_idx;
367 rx_bytes = 0;
368 desc_used = 0;
369 for (pkt = 0; pkt < pkts_to_process; pkt++) {
370 buff_info = (struct octep_vf_rx_buffer *)&oq->buff_info[read_idx];
371 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
372 PAGE_SIZE, DMA_FROM_DEVICE);
373 resp_hw = page_address(buff_info->page);
374 buff_info->page = NULL;
375
376 /* Swap the length field that is in Big-Endian to CPU */
377 buff_info->len = be64_to_cpu(resp_hw->length);
378 if (oct->fw_info.rx_ol_flags) {
379 /* Extended response header is immediately after
380 * response header (resp_hw)
381 */
382 resp_hw_ext = (struct octep_vf_oq_resp_hw_ext *)
383 (resp_hw + 1);
384 buff_info->len -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
385 /* Packet Data is immediately after
386 * extended response header.
387 */
388 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE +
389 OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
390 rx_ol_flags = resp_hw_ext->rx_ol_flags;
391 } else {
392 /* Data is immediately after
393 * Hardware Rx response header.
394 */
395 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE;
396 rx_ol_flags = 0;
397 }
398 rx_bytes += buff_info->len;
399
400 if (buff_info->len <= oq->max_single_buffer_size) {
401 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);
402 skb_reserve(skb, data_offset);
403 skb_put(skb, buff_info->len);
404 read_idx++;
405 desc_used++;
406 if (read_idx == oq->max_count)
407 read_idx = 0;
408 } else {
409 struct skb_shared_info *shinfo;
410 u16 data_len;
411
412 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);
413 skb_reserve(skb, data_offset);
414 /* Head fragment includes response header(s);
415 * subsequent fragments contains only data.
416 */
417 skb_put(skb, oq->max_single_buffer_size);
418 read_idx++;
419 desc_used++;
420 if (read_idx == oq->max_count)
421 read_idx = 0;
422
423 shinfo = skb_shinfo(skb);
424 data_len = buff_info->len - oq->max_single_buffer_size;
425 while (data_len) {
426 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
427 PAGE_SIZE, DMA_FROM_DEVICE);
428 buff_info = (struct octep_vf_rx_buffer *)
429 &oq->buff_info[read_idx];
430 if (data_len < oq->buffer_size) {
431 buff_info->len = data_len;
432 data_len = 0;
433 } else {
434 buff_info->len = oq->buffer_size;
435 data_len -= oq->buffer_size;
436 }
437
438 skb_add_rx_frag(skb, shinfo->nr_frags,
439 buff_info->page, 0,
440 buff_info->len,
441 buff_info->len);
442 buff_info->page = NULL;
443 read_idx++;
444 desc_used++;
445 if (read_idx == oq->max_count)
446 read_idx = 0;
447 }
448 }
449
450 skb->dev = oq->netdev;
451 skb->protocol = eth_type_trans(skb, skb->dev);
452 if (feat & NETIF_F_RXCSUM &&
453 OCTEP_VF_RX_CSUM_VERIFIED(rx_ol_flags))
454 skb->ip_summed = CHECKSUM_UNNECESSARY;
455 else
456 skb->ip_summed = CHECKSUM_NONE;
457 napi_gro_receive(oq->napi, skb);
458 }
459
460 oq->host_read_idx = read_idx;
461 oq->refill_count += desc_used;
462 oq->stats->packets += pkt;
463 oq->stats->bytes += rx_bytes;
464
465 return pkt;
466 }
467
468 /**
469 * octep_vf_oq_process_rx() - Process Rx queue.
470 *
471 * @oq: Octeon Rx queue data structure.
472 * @budget: max number of packets can be processed in one invocation.
473 *
474 * Check for newly received packets and process them.
475 * Keeps checking for new packets until budget is used or no new packets seen.
476 *
477 * Return: number of packets processed.
478 */
octep_vf_oq_process_rx(struct octep_vf_oq * oq,int budget)479 int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget)
480 {
481 u32 pkts_available, pkts_processed, total_pkts_processed;
482 struct octep_vf_device *oct = oq->octep_vf_dev;
483
484 pkts_available = 0;
485 pkts_processed = 0;
486 total_pkts_processed = 0;
487 while (total_pkts_processed < budget) {
488 /* update pending count only when current one exhausted */
489 if (oq->pkts_pending == 0)
490 octep_vf_oq_check_hw_for_pkts(oct, oq);
491 pkts_available = min(budget - total_pkts_processed,
492 oq->pkts_pending);
493 if (!pkts_available)
494 break;
495
496 pkts_processed = __octep_vf_oq_process_rx(oct, oq,
497 pkts_available);
498 oq->pkts_pending -= pkts_processed;
499 total_pkts_processed += pkts_processed;
500 }
501
502 if (oq->refill_count >= oq->refill_threshold) {
503 u32 desc_refilled = octep_vf_oq_refill(oct, oq);
504
505 /* flush pending writes before updating credits */
506 smp_wmb();
507 writel(desc_refilled, oq->pkts_credit_reg);
508 }
509
510 return total_pkts_processed;
511 }
512