1 /***********************license start***********************************
2 * Copyright (c) 2003-2017 Cavium Inc. ([email protected]). All rights
3 * reserved.
4 *
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of Cavium Inc. nor the names of
19 * its contributors may be used to endorse or promote products
20 * derived from this software without specific prior written
21 * permission.
22 *
23 * This Software, including technical data, may be subject to U.S. export
24 * control laws, including the U.S. Export Administration Act and its
25 * associated regulations, and may be subject to export or import
26 * regulations in other countries.
27 *
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
31 * TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
32 * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
33 * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
34 * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
35 * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
36 * QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK
37 * ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
39 #include <bdk.h>
40 #include <malloc.h>
41 #include "libbdk-arch/bdk-csrs-nic.h"
42
43 #define MAX_MTU 9212
44 #define CQ_ENTRIES_QSIZE 0
45 #define CQ_ENTRIES (1024 << CQ_ENTRIES_QSIZE)
46 #define SQ_ENTRIES_QSIZE 0
47 #define SQ_ENTRIES (1024 << SQ_ENTRIES_QSIZE)
48 #define RBDR_ENTRIES_QSIZE 0
49 #define RBDR_ENTRIES (8192 << RBDR_ENTRIES_QSIZE)
50
51 typedef struct
52 {
53 /* VNIC related config */
54 bdk_node_t node : 8; /* Node the NIC is on */
55 bdk_nic_type_t ntype : 8; /* They type of device this NIC is connected to */
56 uint8_t nic_vf; /* NIC VF index number (0 - MAX_VNIC-1) */
57 uint8_t sq; /* Send Queue (SQ) inside NIC VF (0-7) */
58 uint8_t cq; /* Complete Queue (CQ) inside NIC VF (0-7) */
59 uint8_t rq; /* Receive Queue (RQ) inside NIC VF (0-7) */
60 uint8_t rbdr; /* Receive Buffer Descriptor Ring (RBDR) inside NIC VF (0-1) */
61 uint8_t bpid; /* Backpressure ID (0-127) */
62 bdk_if_handle_t handle; /* bdk-if handle associated with this NIC */
63
64 /* Transmit */
65 void * sq_base; /* Pointer to the beginning of the SQ in memory */
66 int sq_loc; /* Location where the next send should go */
67 int sq_available; /* Amount of space left in the queue (fuzzy) */
68 } nic_t;
69
70 typedef struct
71 {
72 void *base;
73 int loc;
74 } nic_rbdr_state_t;
75
76 typedef struct
77 {
78 int num_nic_vf;
79 int next_free_nic_vf;
80 int next_free_cpi;
81 int next_free_rssi;
82 int next_free_bpid;
83 nic_t *nic_map[]; /* Indexed by handle->nic_id */
84 } nic_node_state_t;
85
86 static nic_node_state_t *global_node_state[BDK_NUMA_MAX_NODES];
87 static int global_buffer_size = 0;
88
89 /**
90 * Setup a receive Completion Queue (CQ). CQ can be shared across multiple NICs
91 * to save space. This happens if the NIC has "shares_cq" set.
92 *
93 * @param nic NIC to setup
94 *
95 * @return Zero on success, negative on failure
96 */
vnic_setup_cq(nic_t * nic)97 static int vnic_setup_cq(nic_t *nic)
98 {
99 /* CN88XX pass 1.x had the drop level reset value too low */
100 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_CQM_CFG,
101 c.s.drop_level = 128);
102
103 /* All devices using the same NIC VF use the same CQ */
104 if (nic->handle->index == 0)
105 {
106 BDK_TRACE(NIC, "%s: Setting up CQ(%d, %d)\n", nic->handle->name, nic->nic_vf, nic->cq);
107 /* Note that the completion queue requires 512 byte alignment */
108 void *cq_memory = memalign(512, 512 * CQ_ENTRIES);
109 if (!cq_memory)
110 {
111 bdk_error("%s: Failed to allocate memory for completion queue\n", nic->handle->name);
112 return -1;
113 }
114 /* Configure the completion queue (CQ) */
115 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_CQX_BASE(nic->nic_vf, nic->cq),
116 bdk_ptr_to_phys(cq_memory));
117 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_QSX_CQX_CFG(nic->nic_vf, nic->cq),
118 c.s.ena = 1;
119 c.s.caching = 1;
120 c.s.qsize = CQ_ENTRIES_QSIZE);
121 }
122
123 /* Configure our vnic to send to the CQ */
124 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_SQX_CFG(nic->nic_vf, nic->sq),
125 c.s.cq_qs = nic->nic_vf;
126 c.s.cq_idx = nic->cq);
127 return 0;
128 }
129
130 /**
131 * Add buffers to a receive buffer descriptor ring (RBDR). Note that RBDRs are
132 * shared between NICs using the same CQ.
133 *
134 * @param nic NIC using the RBDR
135 * @param rbdr_free Number of buffers to add
136 */
vnic_fill_receive_buffer(const nic_t * nic,int rbdr_free)137 static void vnic_fill_receive_buffer(const nic_t *nic, int rbdr_free)
138 {
139 int nic_vf = nic->nic_vf;
140 int rbdr = nic->rbdr;
141
142 BDK_CSR_INIT(rbdr_base, nic->node, BDK_NIC_QSX_RBDRX_BASE(nic_vf, rbdr));
143 BDK_CSR_INIT(rbdr_tail, nic->node, BDK_NIC_QSX_RBDRX_TAIL(nic_vf, rbdr));
144 BDK_TRACE(NIC, "%s: In Filling RBDR(%d, %d) base 0x%lx\n", nic->handle->name, nic->nic_vf, nic->rbdr, rbdr_base.u);
145
146 uint64_t *rbdr_ptr = bdk_phys_to_ptr(rbdr_base.u);
147 int loc = rbdr_tail.s.tail_ptr;
148 BDK_TRACE(NIC, "%s: In Filling RBDR(%d, %d) loc %d\n", nic->handle->name, nic->nic_vf, nic->rbdr, loc);
149
150 int added = 0;
151 for (int i = 0; i < rbdr_free; i++)
152 {
153 bdk_if_packet_t packet;
154 if (bdk_if_alloc(&packet, global_buffer_size))
155 {
156 bdk_error("%s: Failed to allocate buffer for RX ring (added %d)\n", nic->handle->name, added);
157 break;
158 }
159 rbdr_ptr[loc] = bdk_cpu_to_le64(packet.packet[0].s.address);
160 BDK_TRACE(NIC, "%s: In Filling RBDR(%d, %d) loc %d = 0x%lx\n", nic->handle->name, nic->nic_vf, nic->rbdr, loc, rbdr_ptr[loc]);
161 loc++;
162 loc &= RBDR_ENTRIES - 1;
163 added++;
164 }
165 BDK_WMB;
166 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_RBDRX_DOOR(nic_vf, rbdr), added);
167 BDK_TRACE(NIC, "%s: In Filling RBDR(%d, %d) added %d\n", nic->handle->name, nic->nic_vf, nic->rbdr, added);
168 }
169
170 /**
171 * Setup a receive buffer descriptor ring (RBDR). Note that NIC share the RBDR if
172 * "share_cq" is set.
173 *
174 * @param nic NIC to setup RBDR for
175 *
176 * @return Zero on success, negative on failure
177 */
vnic_setup_rbdr(nic_t * nic)178 static int vnic_setup_rbdr(nic_t *nic)
179 {
180 bool do_fill;
181
182 /* All devices using the same NIC VF use the same RBDRs. Don't fill them
183 for and ports except the first */
184 if (nic->handle->index)
185 {
186 do_fill = false;
187 }
188 else
189 {
190 BDK_TRACE(NIC, "%s: Setting up RBDR(%d, %d)\n", nic->handle->name, nic->nic_vf, nic->rbdr);
191 void *rbdr_base = memalign(BDK_CACHE_LINE_SIZE, 8 * RBDR_ENTRIES);
192 if (!rbdr_base)
193 {
194 bdk_error("%s: Failed to allocate memory for RBDR\n", nic->handle->name);
195 return -1;
196 }
197 /* Configure the receive buffer ring (RBDR) */
198 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_RBDRX_BASE(nic->nic_vf, nic->rbdr),
199 bdk_ptr_to_phys(rbdr_base));
200 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_QSX_RBDRX_CFG(nic->nic_vf, nic->rbdr),
201 c.s.ena = 1;
202 c.s.ldwb = BDK_USE_DWB;
203 c.s.qsize = RBDR_ENTRIES_QSIZE;
204 c.s.lines = global_buffer_size / BDK_CACHE_LINE_SIZE);
205 do_fill = true;
206 }
207
208 BDK_TRACE(NIC, "%s: Setting up RQ(%d, %d)\n", nic->handle->name, nic->nic_vf, nic->rq);
209 /* Configure our vnic to use the RBDR */
210 /* Connect this RQ to the RBDR. Both the first and next buffers come from
211 the same RBDR */
212 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_RQX_CFG(nic->nic_vf, nic->rq),
213 c.s.caching = 1; /* Allocate to L2 */
214 c.s.cq_qs = nic->nic_vf;
215 c.s.cq_idx = nic->cq;
216 c.s.rbdr_cont_qs = nic->nic_vf;
217 c.s.rbdr_cont_idx = nic->rbdr;
218 c.s.rbdr_strt_qs = nic->nic_vf;
219 c.s.rbdr_strt_idx = nic->rbdr);
220 /* NIC_PF_CQM_CFG is configure to drop everything if the CQ has 128 or
221 less entries available. Start backpressure when we have 256 or less */
222 int cq_bp = 256;
223 int rbdr_bp = 256;
224 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_RQX_BP_CFG(nic->nic_vf, nic->rq),
225 c.s.rbdr_bp_ena = 1;
226 c.s.cq_bp_ena = 1;
227 c.s.rbdr_bp = rbdr_bp * 256 / RBDR_ENTRIES; /* Zero means no buffers, 256 means lots available */
228 c.s.cq_bp = cq_bp * 256 / CQ_ENTRIES; /* Zero means full, 256 means idle */
229 c.s.bpid = nic->bpid);
230 /* Errata (NIC-21269) Limited NIC receive scenario verification */
231 /* RED drop set with pass=drop, so no statistical dropping */
232 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_RQX_DROP_CFG(nic->nic_vf, nic->rq),
233 c.s.rbdr_red = 0;
234 c.s.cq_red = 0;
235 c.s.rbdr_pass = 0; /* Zero means no buffers, 256 means lots available */
236 c.s.rbdr_drop = 0;
237 c.s.cq_pass = 0; /* Zero means full, 256 means idle */
238 c.s.cq_drop = 0);
239
240 if (do_fill)
241 {
242 BDK_TRACE(NIC, "%s: Filling RBDR(%d, %d)\n", nic->handle->name, nic->nic_vf, nic->rbdr);
243 /* We probably don't have enough space to completely fill the RBDR. Use
244 1/8 of the buffers available */
245 int fill_num = bdk_config_get_int(BDK_CONFIG_NUM_PACKET_BUFFERS) / 8;
246 if (CAVIUM_IS_MODEL(CAVIUM_CN83XX)) fill_num = fill_num/3; /* CN83XX has more nics */
247 /* Note that RBDR must leave one spot empty */
248 if (fill_num > RBDR_ENTRIES - 1)
249 fill_num = RBDR_ENTRIES - 1;
250 vnic_fill_receive_buffer(nic, fill_num);
251 }
252
253 return 0;
254 }
255
256 /**
257 * Setup traffic shapping for a NIC. This put the shappers in passthrough mode
258 * where no shapping is applied.
259 *
260 * @param nic NIC to configure shaping for
261 *
262 * @return Zero on success, negative on failure
263 */
vnic_setup_tx_shaping(nic_t * nic)264 static int vnic_setup_tx_shaping(nic_t *nic)
265 {
266 int tl1_index = -1;
267 int tl2_index = -1;
268 int tl3_index = -1;
269 int tl4_index = -1;
270 int nic_chan_e = -1;
271
272 BDK_TRACE(NIC, "%s: Setting up shaping(%d, %d)\n", nic->handle->name, nic->nic_vf, nic->sq);
273
274 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
275 {
276 /* TL1 feeds the DMA engines. One for each BGX */
277 tl1_index = nic->handle->interface;
278 /* TL2 feeds TL1 based on the top/bottom half. Use an independent TL1
279 entry for each BGX port */
280 tl2_index = tl1_index * 32 + nic->handle->index;
281 /* Each block of 4 TL3 feed TL2 */
282 tl3_index = tl2_index * 4;
283 /* Each block of 4 TL4 feed TL3 */
284 tl4_index = tl3_index * 4;
285 nic_chan_e = BDK_NIC_CHAN_E_BGXX_PORTX_CHX(nic->handle->interface, nic->handle->index, 0/*channel*/);
286 }
287 else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
288 {
289 switch (nic->ntype)
290 {
291 case BDK_NIC_TYPE_BGX:
292 tl1_index = BDK_NIC_LMAC_E_BGXX_LMACX(nic->handle->interface, nic->handle->index);
293 nic_chan_e = 0 ; /* Channel is lmac-relative */
294 break;
295 case BDK_NIC_TYPE_LBK:
296 tl1_index = BDK_NIC_LMAC_E_LBKX_CN83XX((nic->handle->interface == 3) ? 1 : 0);
297 nic_chan_e = nic->handle->index; /* Channel is lmac-relative */
298 break;
299 default:
300 bdk_error("%s: Unsupported NIC TYPE %d\n", nic->handle->name, nic->ntype);
301 return -1;
302 }
303 /* TL1 index by NIC_LMAC_E */
304 /* Set in above switch statement */
305 /* TL2 index is software defined, make it the same as TL1 for straight through */
306 tl2_index = tl1_index;
307 /* Each block of 4 TL3 feed TL2. This assumes there are never more than 4 ports per interface */
308 tl3_index = tl2_index * 4 + nic->handle->index;
309 /* TL4 index is the same as TL3, 1:1 hookup */
310 tl4_index = tl3_index;
311 }
312 else if (CAVIUM_IS_MODEL(CAVIUM_CN81XX))
313 {
314 switch (nic->ntype)
315 {
316 case BDK_NIC_TYPE_BGX:
317 tl1_index = BDK_NIC_LMAC_E_BGXX_LMACX(nic->handle->interface, nic->handle->index);
318 nic_chan_e = BDK_NIC_CHAN_E_BGXX_LMACX_CHX(nic->handle->interface, nic->handle->index, 0/*channel*/);
319 break;
320 case BDK_NIC_TYPE_RGMII:
321 tl1_index = BDK_NIC_LMAC_E_RGXX_LMACX(nic->handle->interface, nic->handle->index);
322 nic_chan_e = 0; /* Channel is lmac-relative */
323 break;
324 case BDK_NIC_TYPE_LBK:
325 tl1_index = BDK_NIC_LMAC_E_LBKX_CN81XX(nic->handle->interface);
326 nic_chan_e = nic->handle->index; /* Channel is lmac-relative */
327 break;
328 default:
329 bdk_error("%s: Unsupported NIC TYPE %d\n", nic->handle->name, nic->ntype);
330 return -1;
331 }
332 /* TL1 index by NIC_LMAC_E */
333 /* Set in above switch statement */
334 /* TL2 index is software defined, make it the same as TL1 for straight through */
335 tl2_index = tl1_index;
336 /* Each block of 4 TL3 feed TL2. This assumes there are never more than 4 ports per interface */
337 tl3_index = tl2_index * 4 + nic->handle->index;
338 /* TL4 index is the same as TL3, 1:1 hookup */
339 tl4_index = tl3_index;
340 }
341 else
342 {
343 bdk_error("%s: Unsupported chip (NIC shaping)\n", nic->handle->name);
344 return -1;
345 }
346
347 /* Setup TL2 to TL1 mappings */
348 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL2X_CFG(tl2_index),
349 c.s.rr_quantum = (MAX_MTU+4) / 4);
350 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL2X_PRI(tl2_index),
351 c.s.rr_pri = 0);
352 if (!CAVIUM_IS_MODEL(CAVIUM_CN88XX))
353 {
354 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL2X_LMAC(tl2_index),
355 c.s.lmac = tl1_index);
356 }
357
358 /* TL3 feeds Tl2 */
359 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL3AX_CFG(tl3_index / 4),
360 c.s.tl3a = tl2_index);
361 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL3X_CFG(tl3_index),
362 c.s.rr_quantum = (MAX_MTU+4) / 4);
363 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL3X_CHAN(tl3_index),
364 c.s.chan = nic_chan_e);
365
366 /* TL4 feeds TL3 */
367 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
368 {
369 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL4AX_CFG(tl4_index / 4),
370 c.s.tl4a = tl3_index);
371 }
372 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_TL4X_CFG(tl4_index),
373 c.s.sq_qs = nic->nic_vf;
374 c.s.sq_idx = nic->sq;
375 c.s.rr_quantum = (MAX_MTU+4) / 4);
376
377 /* SQ feeds TL4 */
378 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_SQX_CFG2(nic->nic_vf, nic->sq),
379 c.s.tl4 = tl4_index);
380
381 return 0;
382 }
383
384 /**
385 * Free the buffers in a packet to the RBDR used by the port
386 *
387 * @param priv Determines which RBDR is used
388 * @param packet Packet to put in RBDR
389 */
if_free_to_rbdr(bdk_if_packet_t * packet,nic_rbdr_state_t * vnic_rbdr_state)390 static void if_free_to_rbdr(bdk_if_packet_t *packet, nic_rbdr_state_t *vnic_rbdr_state)
391 {
392 uint64_t *rbdr_ptr = vnic_rbdr_state->base;
393 int loc = vnic_rbdr_state->loc;
394
395 for (int s = 0; s < packet->segments; s++)
396 {
397 /* Make sure we strip off any padding added by the hardware in the address */
398 uint64_t address = packet->packet[s].s.address & -BDK_CACHE_LINE_SIZE;
399 rbdr_ptr[loc] = bdk_cpu_to_le64(address);
400 loc++;
401 loc &= RBDR_ENTRIES - 1;
402 }
403 vnic_rbdr_state->loc = loc;
404 }
405
406 /**
407 * Process a CQ receive entry
408 *
409 * @param node Node containing the CQ
410 * @param vnic_rbdr_state
411 * Current RBDR state for the RBDR connected to the CQ
412 * @param cq_header CQ header to process
413 * @param use_cqe_rx2
414 * True of the CQ will contain an extended CQE_RX2 header
415 *
416 * @return Returns the amount the RBDR doorbell needs to increment
417 */
if_process_complete_rx(int node,nic_rbdr_state_t * vnic_rbdr_state,const union bdk_nic_cqe_rx_s * cq_header,const union bdk_nic_cqe_rx_s * cq_header_le,bool use_cqe_rx2)418 static int if_process_complete_rx(int node, nic_rbdr_state_t *vnic_rbdr_state, const union bdk_nic_cqe_rx_s *cq_header, const union bdk_nic_cqe_rx_s *cq_header_le, bool use_cqe_rx2)
419 {
420 nic_node_state_t *node_state = global_node_state[node];
421 int nic_id = cq_header->s.rq_qs * 8 + cq_header->s.rq_idx;
422
423 bdk_if_packet_t packet;
424 packet.length = cq_header->s.len;
425 packet.segments = cq_header->s.rb_cnt;
426 packet.if_handle = node_state->nic_map[nic_id]->handle;
427 /* Combine the errlev and errop into a single 11 bit number. Errop
428 is 8 bits, so errlev will be in the top byte */
429 packet.rx_error = cq_header->s.errlev;
430 packet.rx_error <<= 8;
431 packet.rx_error |= cq_header->s.errop;
432
433 const uint16_t *rb_sizes = (void*)cq_header_le + 24; /* Offset of RBSZ0 */
434 const uint64_t *rb_addresses = (uint64_t*)(cq_header_le+1);
435 /* Update offset if nic_cqe_rx2_s is used */
436 if (use_cqe_rx2)
437 rb_addresses += sizeof(union bdk_nic_cqe_rx2_s) / 8;
438 int segment_length = 0;
439
440 for (int s = 0; s < packet.segments; s++)
441 {
442 uint64_t addr = bdk_le64_to_cpu(rb_addresses[s]);
443 BDK_PREFETCH(bdk_phys_to_ptr(addr), 0);
444 packet.packet[s].u = addr;
445 packet.packet[s].s.size = bdk_le16_to_cpu(rb_sizes[s]);
446 BDK_TRACE(NIC, " Receive segment size %d address 0x%lx\n", packet.packet[s].s.size, addr);
447 segment_length += packet.packet[s].s.size;
448 }
449
450 /* If we ran out of buffer the packet could be truncated */
451 if (segment_length < packet.length)
452 packet.length = segment_length;
453
454 if (bdk_likely(packet.if_handle))
455 {
456 /* Do RX stats in software as it is fast and I don't really trust
457 the hardware. The hardware tends to count packets that are received
458 and dropped in some weird way. Hopefully the hardware counters
459 looking for drops can find these. It is important that they
460 aren't counted as good */
461 packet.if_handle->stats.rx.packets++;
462 packet.if_handle->stats.rx.octets += packet.length;
463 if (packet.if_handle->flags & BDK_IF_FLAGS_HAS_FCS)
464 packet.if_handle->stats.rx.octets += 4;
465 if (packet.rx_error)
466 packet.if_handle->stats.rx.errors++;
467 bdk_if_dispatch_packet(&packet);
468 }
469 else
470 {
471 bdk_error("Unable to determine interface for NIC %d.%d\n", cq_header->s.rq_qs, cq_header->s.rq_idx);
472 }
473
474 if_free_to_rbdr(&packet, vnic_rbdr_state);
475 return packet.segments;
476 }
477
478 /**
479 * Process all entries in a completion queue (CQ). Note that a CQ is shared
480 * among many ports, so packets will be dispatch for other port handles.
481 *
482 * @param handle Interface handle connected to the CQ
483 *
484 * @return Number of packets received
485 */
if_receive(int unused,void * hand)486 static void if_receive(int unused, void *hand)
487 {
488 const nic_t *nic = hand;
489
490 /* Sadly the hardware team decided to change the meaning of NIC_PF_RX_CFG
491 for chips after CN88XX. This stupid spec change was really hard to
492 find */
493 bool use_cqe_rx2 = !CAVIUM_IS_MODEL(CAVIUM_CN88XX);
494
495 /* Figure out which completion queue we're using */
496 int nic_vf = nic->nic_vf;
497 int rbdr = nic->rbdr;
498 int cq = nic->cq;
499
500 BDK_CSR_INIT(cq_base, nic->node, BDK_NIC_QSX_CQX_BASE(nic_vf, cq));
501 const void *cq_ptr = bdk_phys_to_ptr(cq_base.u);
502
503 /* Find the current CQ location */
504 BDK_CSR_INIT(cq_head, nic->node, BDK_NIC_QSX_CQX_HEAD(nic_vf, cq));
505 int loc = cq_head.s.head_ptr;
506
507 /* Store the RBDR data locally to avoid contention */
508 BDK_CSR_INIT(rbdr_base, nic->node, BDK_NIC_QSX_RBDRX_BASE(nic_vf, rbdr));
509 BDK_CSR_INIT(rbdr_tail, nic->node, BDK_NIC_QSX_RBDRX_TAIL(nic_vf, rbdr));
510 nic_rbdr_state_t vnic_rbdr_state;
511 vnic_rbdr_state.base = bdk_phys_to_ptr(rbdr_base.u);
512 vnic_rbdr_state.loc = rbdr_tail.s.tail_ptr;
513
514 BDK_TRACE(NIC, "%s: Receive thread for CQ(%d, %d) started\n", nic->handle->name, nic->nic_vf, nic->cq);
515
516 while (1)
517 {
518 /* Exit immediately if the CQ is empty */
519 BDK_CSR_INIT(cq_status, nic->node, BDK_NIC_QSX_CQX_STATUS(nic_vf, cq));
520 int pending_count = cq_status.s.qcount;
521 if (bdk_likely(!pending_count))
522 {
523 bdk_wait_usec(1);
524 continue;
525 }
526
527 /* Loop through all pending CQs */
528 int rbdr_doorbell = 0;
529 int count = 0;
530 const union bdk_nic_cqe_rx_s *cq_next = cq_ptr + loc * 512;
531 BDK_TRACE(NIC, "%s: Receive thread CQ(%d, %d): %d pending\n", nic->handle->name, nic->nic_vf, nic->cq, pending_count);
532 while (count < pending_count)
533 {
534 const union bdk_nic_cqe_rx_s *cq_header = cq_next;
535 const union bdk_nic_cqe_rx_s *cq_header_le = cq_header;
536 #if __BYTE_ORDER == __BIG_ENDIAN
537 union bdk_nic_cqe_rx_s cq_be;
538 for (int i = 0; i < 6; i++)
539 cq_be.u[i] = bdk_le64_to_cpu(cq_header_le->u[i]);
540 cq_header = &cq_be;
541 #endif
542 BDK_TRACE(NIC, "%s: Receive HDR[%p] = 0x%lx 0x%lx 0x%lx 0x%lx\n",
543 nic->handle->name, cq_header_le, cq_header->u[0], cq_header->u[1], cq_header->u[2], cq_header->u[3]);
544 loc++;
545 loc &= CQ_ENTRIES - 1;
546 cq_next = cq_ptr + loc * 512;
547 BDK_PREFETCH(cq_next, 0);
548 if (bdk_likely(cq_header->s.cqe_type == BDK_NIC_CQE_TYPE_E_RX))
549 rbdr_doorbell += if_process_complete_rx(nic->node, &vnic_rbdr_state, cq_header, cq_header_le, use_cqe_rx2);
550 else
551 bdk_error("Unsupported CQ header type %d\n", cq_header->s.cqe_type);
552 count++;
553 }
554 /* Ring the RBDR doorbell for all packets */
555 BDK_WMB;
556 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_RBDRX_DOOR(nic_vf, rbdr), rbdr_doorbell);
557 /* Free all the CQs that we've processed */
558 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_CQX_DOOR(nic_vf, cq), count);
559 /* Yield before going through more packets. The low core count chips
560 don't have enough cores to dedicate for TX and RX. This forces
561 sharing under load. If there are enough cores, the yield does
562 nothing */
563 bdk_thread_yield();
564 }
565 }
566
567 /**
568 * Configure NIC for a specific port. This is called for each
569 * port on every interface that connects to NIC.
570 *
571 * @param handle Handle for port to config
572 * @param ntype Type of LMAC this NIC connects to
573 * @param lmac_credits
574 * Size of the LMAC buffer in bytes. Used to configure the number of credits to
575 * setup between the NIC and LMAC
576 *
577 * @return Zero on success, negative on failure
578 */
bdk_nic_port_init(bdk_if_handle_t handle,bdk_nic_type_t ntype,int lmac_credits)579 int bdk_nic_port_init(bdk_if_handle_t handle, bdk_nic_type_t ntype, int lmac_credits)
580 {
581 int nic_chan_idx_e; /* Flow channel for the CPI */
582 bool has_rx_nic = (-1 == handle->pki_channel); /* true when nic rx channel exists - may be BGX or LBK-NIC*/
583 bool has_tx_nic = (-1 == handle->pko_queue); /* true when nic tx channel exists - may be BGX or LBK-NIC*/
584 int nic_intf_e = -1; /* Interface enumeration */
585 int nic_intf_block_e; /* Interface Block ID Enumeration */
586 int nic_lmac_e=-1; /* LMAC enumeration */
587
588 if (global_buffer_size == 0)
589 global_buffer_size = bdk_config_get_int(BDK_CONFIG_PACKET_BUFFER_SIZE);
590
591 if (!has_rx_nic && !has_tx_nic) return 0;
592
593 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
594 {
595 /* Flow here is a compressed NIC_CHAN_E enum value. Flow is bit[8] and
596 bit[6:0] from NIC_CHAN_E. This works out as:
597 bit 7: BGX interface number(0-1)
598 bit 6:4: BGX port number(0-3)
599 bit 3:0: BGX channel on a port (0-15) */
600 nic_chan_idx_e = (handle->interface) ? 0x80 : 0x00;
601 nic_chan_idx_e += handle->index * 16;
602 nic_chan_idx_e += 0; /* channel */
603 nic_intf_e = BDK_NIC_INTF_E_BGXX(handle->interface);
604 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_BGXX_BLOCK(handle->interface);
605 nic_lmac_e = BDK_NIC_LMAC_E_BGXX_LMACX(handle->interface, handle->index);
606 }
607 else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
608 {
609 switch (ntype)
610 {
611 case BDK_NIC_TYPE_BGX:
612 nic_chan_idx_e = BDK_NIC_CHAN_IDX_E_BGXX_LMACX_CHX(handle->interface, handle->index, 0/*channel*/);
613 nic_intf_e = BDK_NIC_INTF_E_BGXX(handle->interface);
614 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_BGXX(handle->interface);
615 nic_lmac_e = BDK_NIC_LMAC_E_BGXX_LMACX(handle->interface, handle->index);
616 break;
617 case BDK_NIC_TYPE_LBK:
618 nic_chan_idx_e = BDK_NIC_CHAN_IDX_E_LBKX_CHX_CN83XX((handle->interface == 3) ? 1 : 0, handle->index);
619 // rx interface
620 if (3 == handle->interface) {
621 nic_intf_e = BDK_NIC_INTF_E_LBKX_CN83XX(1);
622 } else if (2 == handle->interface) {
623 nic_intf_e = BDK_NIC_INTF_E_LBKX_CN83XX(0);
624 }
625 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_LBKX(handle->interface);
626 // tx interface
627 if (3 == handle->interface) {
628 nic_lmac_e = BDK_NIC_LMAC_E_LBKX_CN83XX(1);
629 } else if (1 == handle->interface) {
630 nic_lmac_e = BDK_NIC_LMAC_E_LBKX_CN83XX(0);
631 }
632 break;
633 default:
634 bdk_error("%s: Unsupported NIC TYPE %d\n", handle->name, ntype);
635 return -1;
636 }
637 }
638 else if (CAVIUM_IS_MODEL(CAVIUM_CN81XX))
639 {
640 switch (ntype)
641 {
642 case BDK_NIC_TYPE_BGX:
643 nic_chan_idx_e = BDK_NIC_CHAN_IDX_E_BGXX_LMACX_CHX(handle->interface, handle->index, 0/*channel*/);
644 nic_intf_e = BDK_NIC_INTF_E_BGXX(handle->interface);
645 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_BGXX(handle->interface);
646 nic_lmac_e = BDK_NIC_LMAC_E_BGXX_LMACX(handle->interface, handle->index);
647 break;
648 case BDK_NIC_TYPE_RGMII:
649 nic_chan_idx_e = BDK_NIC_CHAN_IDX_E_RGXX_LMACX_CHX(handle->interface, handle->index, 0/*channel*/);
650 nic_intf_e = BDK_NIC_INTF_E_RGXX(handle->index);
651 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_BGXX(handle->interface + 2);
652 nic_lmac_e = BDK_NIC_LMAC_E_RGXX_LMACX(handle->interface, handle->index);
653 break;
654 case BDK_NIC_TYPE_LBK:
655 nic_chan_idx_e = BDK_NIC_CHAN_IDX_E_LBKX_CHX_CN81XX(handle->interface, handle->index);
656 nic_intf_e = BDK_NIC_INTF_E_LBKX_CN81XX(handle->interface);
657 nic_intf_block_e = BDK_NIC_INTF_BLOCK_E_LBKX(handle->interface);
658 nic_lmac_e = BDK_NIC_LMAC_E_LBKX_CN81XX(handle->interface);
659 break;
660 default:
661 bdk_error("%s: Unsupported NIC TYPE %d\n", handle->name, ntype);
662 return -1;
663 }
664 }
665 else
666 {
667 bdk_error("%s: Unsupported chip (NIC init)\n", handle->name);
668 return -1;
669 }
670
671 /* Make sure the node global state has been allocated */
672 if (global_node_state[handle->node] == NULL)
673 {
674 int num_nic_vf;
675 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
676 {
677 /* NIC_PF_CONST1 didn't exist on this chip */
678 num_nic_vf = 128;
679 }
680 else
681 {
682 BDK_CSR_INIT(nic_pf_const1, handle->node, BDK_NIC_PF_CONST1);
683 num_nic_vf = nic_pf_const1.s.vnics;
684 }
685 global_node_state[handle->node] = calloc(1, sizeof(nic_node_state_t) + sizeof(handle) * num_nic_vf * 8);
686 if (global_node_state[handle->node] == NULL)
687 {
688 bdk_error("N%d.NIC: Failed to allocate node state\n", handle->node);
689 return -1;
690 }
691 global_node_state[handle->node]->num_nic_vf = num_nic_vf;
692 }
693 nic_node_state_t *node_state = global_node_state[handle->node];
694
695 /* See if we have a free VF */
696 if (!handle->index && (node_state->next_free_nic_vf >= node_state->num_nic_vf))
697 {
698 bdk_error("N%d.NIC: Ran out of NIC VFs\n", handle->node);
699 return -1;
700 }
701
702 /* VNIC setup requirements
703 The code in this file makes the following assumptions:
704 1) One RBDR for each CQ. No locking is done on RBDR
705 2) A CQ can be shared across multiple ports, saving space as the
706 cost of performance.
707 3) One SQ per physical port, no locking on TX
708 4) One RQ per physical port, many RQ may share RBDR/CQ
709
710 Current setup without DRAM:
711 1) One NIC VF is used for an entire interface (BGX, LBK). The variable
712 nic_vf represents the NIC virtual function.
713 2) SQs are allocated one per port. SQ index equals handle->index
714 3) RQs are allocated one per port. RQ index equals handle->index
715 4) One CQ is allcoated per entire interface, using index 0
716 5) One RBDR is used for the CQ, index 0
717
718 Current setup with DRAM:
719 FIXME: Same as without DRAM. There are not enough RBDR to have
720 independent CQs without locking.
721 */
722 void *sq_memory = NULL;
723 if (has_tx_nic) {
724 sq_memory = memalign(BDK_CACHE_LINE_SIZE, 16 * SQ_ENTRIES);
725 if (!sq_memory)
726 {
727 bdk_error("%s: Unable to allocate queues\n", handle->name);
728 return -1;
729 }
730 }
731 nic_t *nic = calloc(1, sizeof(nic_t));
732 if (!nic)
733 {
734 if (sq_memory) free(sq_memory);
735 bdk_error("%s: Unable to NIC state\n", handle->name);
736 return -1;
737 }
738
739 /* Fill in the various NIC indexes */
740 nic->node = handle->node;
741 nic->ntype = ntype;
742 if (handle->index)
743 nic->nic_vf = node_state->next_free_nic_vf - 1; /* reuse last one */
744 else
745 nic->nic_vf = node_state->next_free_nic_vf++; /* New nic */
746 nic->sq = handle->index;
747 nic->cq = 0;
748 nic->rq = handle->index;
749 nic->rbdr = 0;
750 nic->bpid = node_state->next_free_bpid++;
751 nic->handle = handle;
752 BDK_TRACE(NIC, "%s: Creating NIC(%d, sq=%d, cq=%d, rq=%d, rbdr=%d, bpid=%d)\n",
753 nic->handle->name, nic->nic_vf, nic->sq, nic->cq, nic->rq, nic->rbdr, nic->bpid);
754
755 /* Connect this NIC to the handle */
756 handle->nic_id = nic->nic_vf * 8 + nic->rq;
757 node_state->nic_map[handle->nic_id] = nic;
758
759 /* Enable global BP state updates */
760 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_BP_CFG,
761 c.s.bp_poll_ena = 1;
762 c.s.bp_poll_dly = 3);
763
764 /* Enable interface level backpresure */
765 if (-1 != nic_intf_e) {
766 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_INTFX_BP_CFG(nic_intf_e),
767 c.s.bp_ena = 1;
768 c.s.bp_type = ((nic->ntype == BDK_NIC_TYPE_BGX) ||
769 (nic->ntype == BDK_NIC_TYPE_RGMII)) ? 0 : 1; /* 0=BGX, 1=LBK/TNS */
770 c.s.bp_id = nic_intf_block_e);
771 }
772 if (has_tx_nic) {
773 /* Configure the submit queue (SQ) */
774 nic->sq_base = sq_memory;
775 nic->sq_loc = 0;
776 nic->sq_available = SQ_ENTRIES;
777 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_SQX_BASE(nic->nic_vf, nic->sq),
778 bdk_ptr_to_phys(sq_memory));
779 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_QSX_SQX_CFG(nic->nic_vf, nic->sq),
780 if (!CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
781 c.s.cq_limit = 1;
782 c.s.ena = 1;
783 c.s.ldwb = BDK_USE_DWB;
784 c.s.qsize = SQ_ENTRIES_QSIZE);
785 }
786 int cpi=0;
787 int rssi=0;
788 if (has_rx_nic) {
789 /* Configure the receive queue (RQ) */
790 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_QSX_RQ_GEN_CFG(nic->nic_vf),
791 c.s.vlan_strip = 0;
792 c.s.len_l4 = 0;
793 c.s.len_l3 = 0;
794 c.s.csum_l4 = 0;
795 c.s.ip6_udp_opt = 0;
796 c.s.splt_hdr_ena = 0;
797 c.s.cq_hdr_copy = 0;
798 c.s.max_tcp_reass = 0;
799 c.s.cq_pkt_size = 0;
800 c.s.later_skip = 0;
801 c.s.first_skip = 0);
802 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_QSX_RQX_CFG(nic->nic_vf, nic->rq),
803 c.s.ena = 1;
804 c.s.tcp_ena = 0);
805
806 cpi = node_state->next_free_cpi++; /* Allocate a new Channel Parse Index (CPI) */
807 rssi = node_state->next_free_rssi++;/* Allocate a new Receive-Side Scaling Index (RSSI) */
808 /* NIC_CHAN_E hard mapped to "flow". Flow chooses the CPI */
809
810 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_CHANX_RX_CFG(nic_chan_idx_e),
811 c.s.cpi_alg = BDK_NIC_CPI_ALG_E_NONE;
812 c.s.cpi_base = cpi);
813 /* Setup backpressure */
814 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_CHANX_RX_BP_CFG(nic_chan_idx_e),
815 c.s.ena = 1;
816 c.s.bpid = nic->bpid);
817 }
818 if ( has_tx_nic) {
819 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_CHANX_TX_CFG(nic_chan_idx_e),
820 c.s.bp_ena = 1);
821 }
822
823 if (has_rx_nic) {
824 /* CPI is the output of the above alogrithm, this is used to lookup the
825 VNIC for receive and RSSI */
826 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_CPIX_CFG(cpi),
827 c.cn88xxp1.vnic = nic->nic_vf; /* TX and RX use the same VNIC */
828 c.cn88xxp1.rss_size = 0; /* RSS hash is disabled */
829 c.s.padd = 0; /* Used if we have multiple channels per port */
830 c.cn88xxp1.rssi_base = rssi); /* Base RSSI */
831
832 if (!CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
833 {
834 /* CN88XX pass 2 moved some fields to a different CSR */
835 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_MPIX_CFG(cpi),
836 c.s.vnic = nic->nic_vf; /* TX and RX use the same VNIC */
837 c.s.rss_size = 0; /* RSS hash is disabled */
838 c.s.rssi_base = rssi); /* Base RSSI */
839 }
840
841 /* The RSSI is used to determine which Receive Queue (RQ) we use */
842 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_RSSIX_RQ(rssi),
843 c.s.rq_qs = nic->nic_vf;
844 c.s.rq_idx = nic->rq);
845 /* Set the min and max packet size. PKND comes from BGX. It is always zero
846 for now */
847 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_PKINDX_CFG(handle->pknd),
848 c.s.lenerr_en = 0;
849 c.s.minlen = 0;
850 c.s.maxlen = 65535);
851 }
852
853 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
854 {
855 /* Bypass the TNS */
856 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_INTFX_SEND_CFG(handle->interface),
857 c.s.tns_nonbypass = 0;
858 c.s.block = 0x8 + handle->interface);
859 }
860
861 /* Errata (NIC-21858) If NIC_PF_QS()_CFG ENA is set after RRM enabled...RRM breaks */
862 /* Do global vnic init */
863 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_QSX_CFG(nic->nic_vf),
864 c.s.ena = 1;
865 c.s.vnic = nic->nic_vf);
866
867 if (has_tx_nic && vnic_setup_tx_shaping(nic))
868 return -1;
869
870 /* Completion queue may be used by both tx and rx.
871 ** Define it even if only one of rx/tx is in use
872 */
873 if (vnic_setup_cq(nic))
874 return -1;
875 /* RBDR is defined regardless of rx_nic to avoid possible backpressure */
876 if ( vnic_setup_rbdr(nic))
877 return -1;
878
879 /* Program LMAC credits */
880 if ((has_tx_nic) && (-1 != nic_lmac_e)) {
881 int credit;
882 if ((BDK_NIC_TYPE_LBK == nic->ntype) && CAVIUM_IS_MODEL(CAVIUM_CN83XX) )
883 credit = 512; /* HRM guidance */
884 else
885 credit = (lmac_credits - MAX_MTU) / 16;
886 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_LMACX_CREDIT(nic_lmac_e),
887 c.s.cc_unit_cnt = credit;
888 c.s.cc_packet_cnt = 0x1ff;
889 c.s.cc_enable = 1);
890
891 /* Pad packets to 60 bytes, 15 32bit words (before FCS) */
892 if (nic->ntype != BDK_NIC_TYPE_LBK)
893 BDK_CSR_MODIFY(c, nic->node, BDK_NIC_PF_LMACX_CFG(nic_lmac_e),
894 c.s.min_pkt_size = 15);
895 }
896 /* Create a receive thread if this handle has its own CQ/RBDR */
897 if (handle->index == 0)
898 {
899 /* FIXME
900 * At this time thread monitors both CQ and RBDR and uses it only for receive
901 * Setting up RBDR for tx only nics is wasteful.
902 * When nic_tx in bdk starts using CQ, thread needs to change
903 */
904 if (has_rx_nic && bdk_thread_create(nic->node, 0, if_receive, 0, nic, 0))
905 {
906 bdk_error("%s: Failed to allocate receive thread\n", handle->name);
907 return -1;
908 }
909 }
910
911 return 0;
912 }
913
914 /**
915 * Send a packet
916 *
917 * @param handle Handle of port to send on
918 * @param packet Packet to send
919 *
920 * @return Zero on success, negative on failure
921 */
bdk_nic_transmit(bdk_if_handle_t handle,const bdk_if_packet_t * packet)922 int bdk_nic_transmit(bdk_if_handle_t handle, const bdk_if_packet_t *packet)
923 {
924 /* The SQ can't be filled completely as it reguires at least one free
925 entry so the head and pointer don't look like empty. SQ_SLOP is the
926 amount of SQ space we reserve to make sure of this */
927 const int SQ_SLOP = 1;
928 const nic_node_state_t *node_state = global_node_state[handle->node];
929 nic_t *nic = node_state->nic_map[handle->nic_id];
930 BDK_TRACE(NIC, "%s: Transmit packet of %d bytes, %d segments\n",
931 nic->handle->name, packet->length, packet->segments);
932
933 /* Update the SQ available if we're out of space. The NIC should have sent
934 packets, making more available. This allows us to only read the STATUS
935 CSR when really necessary, normally using the L1 cached value */
936 if (nic->sq_available < packet->segments + 1 + SQ_SLOP)
937 {
938 BDK_CSR_INIT(sq_status, nic->node, BDK_NIC_QSX_SQX_STATUS(nic->nic_vf, nic->sq));
939 nic->sq_available = SQ_ENTRIES - sq_status.s.qcount;
940 /* Re-Check for space. A packets is a header plus its segments */
941 if (nic->sq_available < packet->segments + 1 + SQ_SLOP)
942 {
943 BDK_TRACE(NIC, "%s: Transmit fail, queue full\n", nic->handle->name);
944 return -1;
945 }
946 }
947
948 /* Build the command */
949 void *sq_ptr = nic->sq_base;
950 int loc = nic->sq_loc;
951 union bdk_nic_send_hdr_s send_hdr;
952 send_hdr.u[0] = 0;
953 send_hdr.u[1] = 0;
954 send_hdr.s.subdc = BDK_NIC_SEND_SUBDC_E_HDR;
955 send_hdr.s.subdcnt = packet->segments;
956 send_hdr.s.total = packet->length;
957 switch (packet->packet_type)
958 {
959 case BDK_IF_TYPE_UNKNOWN:
960 break;
961 case BDK_IF_TYPE_UDP4:
962 send_hdr.s.ckl3 = 1; /* L3 - IPv4 checksum enable */
963 send_hdr.s.l3ptr = 14; /* L2 header is 14 bytes */
964 send_hdr.s.ckl4 = BDK_NIC_SEND_CKL4_E_UDP; /* L4 - UDP checksum enable */
965 send_hdr.s.l4ptr = 14 + 20; /* 14 bytes L2 + 20 bytes IPv4 */
966 break;
967 case BDK_IF_TYPE_TCP4:
968 send_hdr.s.ckl3 = 1; /* L3 - IPv4 checksum enable */
969 send_hdr.s.l3ptr = 14; /* L2 header is 14 bytes */
970 send_hdr.s.ckl4 = BDK_NIC_SEND_CKL4_E_TCP; /* L4 - TCP checksum enable */
971 send_hdr.s.l4ptr = 14 + 20; /* 14 bytes L2 + 20 bytes IPv4 */
972 if (packet->mtu)
973 {
974 int headers = 14 + 20 + 20;
975 send_hdr.s.tso = 1; /* Use TCP offload */
976 send_hdr.s.tso_sb = headers; /* 14 bytes L2 + 20 bytes IPv4, 20 bytes TCP */
977 send_hdr.s.tso_mps = packet->mtu - headers; /* Max TCP data payload size */
978 }
979 break;
980 }
981 volatile uint64_t *wptr = (uint64_t *)(sq_ptr + loc * 16);
982 wptr[0] = bdk_cpu_to_le64(send_hdr.u[0]);
983 wptr[1] = bdk_cpu_to_le64(send_hdr.u[1]);
984 BDK_TRACE(NIC, "%s: Transmit HDR[%p] = 0x%lx 0x%lx\n",
985 nic->handle->name, sq_ptr + loc * 16, send_hdr.u[0], send_hdr.u[1]);
986 loc++;
987 loc &= SQ_ENTRIES - 1;
988 for (int s = 0; s < packet->segments; s++)
989 {
990 union bdk_nic_send_gather_s gather;
991 gather.u[0] = 0;
992 gather.u[1] = 0;
993 gather.s.addr = packet->packet[s].s.address;
994 gather.s.subdc = BDK_NIC_SEND_SUBDC_E_GATHER;
995 gather.s.ld_type = (BDK_USE_DWB) ? BDK_NIC_SEND_LD_TYPE_E_LDWB : BDK_NIC_SEND_LD_TYPE_E_LDD;
996 gather.s.size = packet->packet[s].s.size;
997 wptr = (uint64_t *)(sq_ptr + loc * 16);
998 wptr[0] = bdk_cpu_to_le64(gather.u[0]);
999 wptr[1] = bdk_cpu_to_le64(gather.u[1]);
1000 BDK_TRACE(NIC, "%s: Transmit Gather[%p] = 0x%lx 0x%lx\n",
1001 nic->handle->name, sq_ptr + loc * 16, gather.u[0], gather.u[1]);
1002 loc++;
1003 loc &= SQ_ENTRIES - 1;
1004 }
1005
1006 BDK_WMB;
1007
1008 /* Ring the doorbell */
1009 BDK_CSR_WRITE(nic->node, BDK_NIC_QSX_SQX_DOOR(nic->nic_vf, nic->sq),
1010 packet->segments + 1);
1011 BDK_TRACE(NIC, "%s: Transmit Doorbell %d\n", nic->handle->name, packet->segments + 1);
1012
1013 /* Update our cached state */
1014 nic->sq_available -= packet->segments + 1;
1015 nic->sq_loc = loc;
1016 if (handle->iftype != BDK_IF_BGX) {
1017 /* Update stats as we do them in software for non-BGX */
1018 handle->stats.tx.packets++;
1019 handle->stats.tx.octets += packet->length;
1020 if (handle->flags & BDK_IF_FLAGS_HAS_FCS)
1021 handle->stats.tx.octets += 4;
1022 }
1023 return 0;
1024 }
1025
1026 /**
1027 * Get the current TX queue depth. Note that this operation may be slow
1028 * and adversly affect packet IO performance.
1029 *
1030 * @param handle Port to check
1031 *
1032 * @return Depth of the queue in packets
1033 */
bdk_nic_get_queue_depth(bdk_if_handle_t handle)1034 int bdk_nic_get_queue_depth(bdk_if_handle_t handle)
1035 {
1036 const nic_node_state_t *node_state = global_node_state[handle->node];
1037 const nic_t *nic = node_state->nic_map[handle->nic_id];
1038 BDK_CSR_INIT(sq_status, nic->node, BDK_NIC_QSX_SQX_STATUS(nic->nic_vf, nic->sq));
1039 return sq_status.s.qcount;
1040 }
1041
1042 /**
1043 * Query NIC and fill in the transmit stats for the supplied
1044 * interface handle.
1045 *
1046 * @param handle Port handle
1047 */
bdk_nic_fill_tx_stats(bdk_if_handle_t handle)1048 void bdk_nic_fill_tx_stats(bdk_if_handle_t handle)
1049 {
1050 const int vnic = handle->nic_id >> 3;
1051
1052 /* Transmit stats are done in software due to CN81XX not having enough NICs */
1053
1054 /* Note drops are shared across a BGX. People will be confused */
1055 BDK_CSR_INIT(drps, handle->node, BDK_NIC_VNICX_TX_STATX(vnic, BDK_NIC_STAT_VNIC_TX_E_TX_DROP));
1056 handle->stats.tx.dropped_packets = bdk_update_stat_with_overflow(drps.u, handle->stats.tx.dropped_packets, 48);
1057 /* Dropped Octets are not available */
1058 }
1059
1060 /**
1061 * Query NIC and fill in the receive stats for the supplied
1062 * interface handle.
1063 *
1064 * @param handle Port handle
1065 */
bdk_nic_fill_rx_stats(bdk_if_handle_t handle)1066 void bdk_nic_fill_rx_stats(bdk_if_handle_t handle)
1067 {
1068 /* Account for RX FCS */
1069 const int bytes_off_rx = (handle->flags & BDK_IF_FLAGS_HAS_FCS) ? 4 : 0;
1070 const int vnic = handle->nic_id >> 3;
1071
1072 /* Note stats are shared across a BGX. People will be confused */
1073
1074 /* Read the RX statistics. These do not include the ethernet FCS */
1075 BDK_CSR_INIT(rx_red, handle->node, BDK_NIC_VNICX_RX_STATX(vnic, BDK_NIC_STAT_VNIC_RX_E_RX_RED));
1076 BDK_CSR_INIT(rx_red_octets, handle->node, BDK_NIC_VNICX_RX_STATX(vnic, BDK_NIC_STAT_VNIC_RX_E_RX_RED_OCTS));
1077 BDK_CSR_INIT(rx_ovr, handle->node, BDK_NIC_VNICX_RX_STATX(vnic, BDK_NIC_STAT_VNIC_RX_E_RX_ORUN));
1078 BDK_CSR_INIT(rx_ovr_octets, handle->node, BDK_NIC_VNICX_RX_STATX(vnic, BDK_NIC_STAT_VNIC_RX_E_RX_ORUN_OCTS));
1079 uint64_t drops = rx_red.u + rx_ovr.u;
1080 uint64_t drop_octets = rx_red_octets.u + rx_ovr_octets.u;
1081
1082 /* Drop and error counters */
1083 handle->stats.rx.dropped_octets -= handle->stats.rx.dropped_packets * bytes_off_rx;
1084 handle->stats.rx.dropped_octets = bdk_update_stat_with_overflow(drop_octets, handle->stats.rx.dropped_octets, 48);
1085 handle->stats.rx.dropped_packets = bdk_update_stat_with_overflow(drops, handle->stats.rx.dropped_packets, 48);
1086 handle->stats.rx.dropped_octets += handle->stats.rx.dropped_packets * bytes_off_rx;
1087
1088 /* Normal RX stats are done by software on receive */
1089 }
1090
1091