1 /*
2 * Copyright (c) 2015 Travis Geiselbrecht
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include <dev/virtio/net.h>
24
25 #include <debug.h>
26 #include <assert.h>
27 #include <trace.h>
28 #include <compiler.h>
29 #include <list.h>
30 #include <string.h>
31 #include <err.h>
32 #include <kernel/thread.h>
33 #include <kernel/event.h>
34 #include <kernel/spinlock.h>
35 #include <kernel/vm.h>
36 #include <lib/pktbuf.h>
37 #include <lib/minip.h>
38
39 #define LOCAL_TRACE 0
40
41 struct virtio_net_config {
42 uint8_t mac[6];
43 uint16_t status;
44 uint16_t max_virtqueue_pairs;
45 } __PACKED;
46
47 struct virtio_net_hdr {
48 uint8_t flags;
49 uint8_t gso_type;
50 uint16_t hdr_len;
51 uint16_t gso_size;
52 uint16_t csum_start;
53 uint16_t csum_offset;
54 uint16_t num_buffers; // unused in tx
55 } __PACKED;
56
57 #define VIRTIO_NET_F_CSUM (1<<0)
58 #define VIRTIO_NET_F_GUEST_CSUM (1<<1)
59 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS (1<<2)
60 #define VIRTIO_NET_F_MAC (1<<5)
61 #define VIRTIO_NET_F_GSO (1<<6)
62 #define VIRTIO_NET_F_GUEST_TSO4 (1<<7)
63 #define VIRTIO_NET_F_GUEST_TSO6 (1<<8)
64 #define VIRTIO_NET_F_GUEST_ECN (1<<9)
65 #define VIRTIO_NET_F_GUEST_UFO (1<<10)
66 #define VIRTIO_NET_F_HOST_TSO4 (1<<11)
67 #define VIRTIO_NET_F_HOST_TSO6 (1<<12)
68 #define VIRTIO_NET_F_HOST_ECN (1<<13)
69 #define VIRTIO_NET_F_HOST_UFO (1<<14)
70 #define VIRTIO_NET_F_MRG_RXBUF (1<<15)
71 #define VIRTIO_NET_F_STATUS (1<<16)
72 #define VIRTIO_NET_F_CTRL_VQ (1<<17)
73 #define VIRTIO_NET_F_CTRL_RX (1<<18)
74 #define VIRTIO_NET_F_CTRL_VLAN (1<<19)
75 #define VIRTIO_NET_F_GUEST_ANNOUNCE (1<<21)
76 #define VIRTIO_NET_F_MQ (1<<22)
77 #define VIRTIO_NET_F_CTRL_MAC_ADDR (1<<23)
78
79 #define VIRTIO_NET_S_LINK_UP (1<<0)
80 #define VIRTIO_NET_S_ANNOUNCE (1<<1)
81
82 #define TX_RING_SIZE 16
83 #define RX_RING_SIZE 16
84
85 #define RING_RX 0
86 #define RING_TX 1
87
88 #define VIRTIO_NET_MSS 1514
89
90 struct virtio_net_dev {
91 struct virtio_device *dev;
92 bool started;
93
94 struct virtio_net_config *config;
95
96 spin_lock_t lock;
97 event_t rx_event;
98
99 /* list of active tx/rx packets to be freed at irq time */
100 pktbuf_t *pending_tx_packet[TX_RING_SIZE];
101 pktbuf_t *pending_rx_packet[RX_RING_SIZE];
102
103 uint tx_pending_count;
104 struct list_node completed_rx_queue;
105 };
106
107 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e);
108 static int virtio_net_rx_worker(void *arg);
109 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p);
110
111 // XXX remove need for this
112 static struct virtio_net_dev *the_ndev;
113
dump_feature_bits(uint32_t feature)114 static void dump_feature_bits(uint32_t feature)
115 {
116 printf("virtio-net host features (0x%x):", feature);
117 if (feature & VIRTIO_NET_F_CSUM) printf(" CSUM");
118 if (feature & VIRTIO_NET_F_GUEST_CSUM) printf(" GUEST_CSUM");
119 if (feature & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) printf(" CTRL_GUEST_OFFLOADS");
120 if (feature & VIRTIO_NET_F_MAC) printf(" MAC");
121 if (feature & VIRTIO_NET_F_GSO) printf(" GSO");
122 if (feature & VIRTIO_NET_F_GUEST_TSO4) printf(" GUEST_TSO4");
123 if (feature & VIRTIO_NET_F_GUEST_TSO6) printf(" GUEST_TSO6");
124 if (feature & VIRTIO_NET_F_GUEST_ECN) printf(" GUEST_ECN");
125 if (feature & VIRTIO_NET_F_GUEST_UFO) printf(" GUEST_UFO");
126 if (feature & VIRTIO_NET_F_HOST_TSO4) printf(" HOST_TSO4");
127 if (feature & VIRTIO_NET_F_HOST_TSO6) printf(" HOST_TSO6");
128 if (feature & VIRTIO_NET_F_HOST_ECN) printf(" HOST_ECN");
129 if (feature & VIRTIO_NET_F_HOST_UFO) printf(" HOST_UFO");
130 if (feature & VIRTIO_NET_F_MRG_RXBUF) printf(" MRG_RXBUF");
131 if (feature & VIRTIO_NET_F_STATUS) printf(" STATUS");
132 if (feature & VIRTIO_NET_F_CTRL_VQ) printf(" CTRL_VQ");
133 if (feature & VIRTIO_NET_F_CTRL_RX) printf(" CTRL_RX");
134 if (feature & VIRTIO_NET_F_CTRL_VLAN) printf(" CTRL_VLAN");
135 if (feature & VIRTIO_NET_F_GUEST_ANNOUNCE) printf(" GUEST_ANNOUNCE");
136 if (feature & VIRTIO_NET_F_MQ) printf(" MQ");
137 if (feature & VIRTIO_NET_F_CTRL_MAC_ADDR) printf(" CTRL_MAC_ADDR");
138 printf("\n");
139 }
140
virtio_net_init(struct virtio_device * dev,uint32_t host_features)141 status_t virtio_net_init(struct virtio_device *dev, uint32_t host_features)
142 {
143 LTRACEF("dev %p, host_features 0x%x\n", dev, host_features);
144
145 /* allocate a new net device */
146 struct virtio_net_dev *ndev = calloc(1, sizeof(struct virtio_net_dev));
147 if (!ndev)
148 return ERR_NO_MEMORY;
149
150 ndev->dev = dev;
151 dev->priv = ndev;
152 ndev->started = false;
153
154 ndev->lock = SPIN_LOCK_INITIAL_VALUE;
155 event_init(&ndev->rx_event, false, EVENT_FLAG_AUTOUNSIGNAL);
156 list_initialize(&ndev->completed_rx_queue);
157
158 ndev->config = (struct virtio_net_config *)dev->config_ptr;
159
160 /* ack and set the driver status bit */
161 virtio_status_acknowledge_driver(dev);
162
163 // XXX check features bits and ack/nak them
164 dump_feature_bits(host_features);
165
166 /* set our irq handler */
167 dev->irq_driver_callback = &virtio_net_irq_driver_callback;
168
169 /* set DRIVER_OK */
170 virtio_status_driver_ok(dev);
171
172 /* allocate a pair of virtio rings */
173 virtio_alloc_ring(dev, RING_RX, RX_RING_SIZE); // rx
174 virtio_alloc_ring(dev, RING_TX, TX_RING_SIZE); // tx
175
176 the_ndev = ndev;
177
178 return NO_ERROR;
179 }
180
virtio_net_start(void)181 status_t virtio_net_start(void)
182 {
183 if (the_ndev->started)
184 return ERR_ALREADY_STARTED;
185
186 the_ndev->started = true;
187
188 /* start the rx worker thread */
189 thread_resume(thread_create("virtio_net_rx", &virtio_net_rx_worker, (void *)the_ndev, HIGH_PRIORITY, DEFAULT_STACK_SIZE));
190
191 /* queue up a bunch of rxes */
192 for (uint i = 0; i < RX_RING_SIZE - 1; i++) {
193 pktbuf_t *p = pktbuf_alloc();
194 if (p) {
195 virtio_net_queue_rx(the_ndev, p);
196 }
197 }
198
199 return NO_ERROR;
200 }
201
virtio_net_queue_tx_pktbuf(struct virtio_net_dev * ndev,pktbuf_t * p2)202 static status_t virtio_net_queue_tx_pktbuf(struct virtio_net_dev *ndev, pktbuf_t *p2)
203 {
204 struct virtio_device *vdev = ndev->dev;
205
206 uint16_t i;
207 pktbuf_t *p;
208
209 DEBUG_ASSERT(ndev);
210
211 p = pktbuf_alloc();
212 if (!p)
213 return ERR_NO_MEMORY;
214
215 /* point our header to the base of the first pktbuf */
216 struct virtio_net_hdr *hdr = pktbuf_append(p, sizeof(struct virtio_net_hdr) - 2);
217 memset(hdr, 0, p->dlen);
218
219 spin_lock_saved_state_t state;
220 spin_lock_irqsave(&ndev->lock, state);
221
222 /* only queue if we have enough tx descriptors */
223 if (ndev->tx_pending_count + 2 > TX_RING_SIZE)
224 goto nodesc;
225
226 /* allocate a chain of descriptors for our transfer */
227 struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_TX, 2, &i);
228 if (!desc) {
229 spin_unlock_irqrestore(&ndev->lock, state);
230
231 nodesc:
232 TRACEF("out of virtio tx descriptors, tx_pending_count %u\n", ndev->tx_pending_count);
233 pktbuf_free(p, true);
234
235 return ERR_NO_MEMORY;
236 }
237
238 ndev->tx_pending_count += 2;
239
240 /* save a pointer to our pktbufs for the irq handler to free */
241 LTRACEF("saving pointer to pkt in index %u and %u\n", i, desc->next);
242 DEBUG_ASSERT(ndev->pending_tx_packet[i] == NULL);
243 DEBUG_ASSERT(ndev->pending_tx_packet[desc->next] == NULL);
244 ndev->pending_tx_packet[i] = p;
245 ndev->pending_tx_packet[desc->next] = p2;
246
247 /* set up the descriptor pointing to the header */
248 desc->addr = pktbuf_data_phys(p);
249 desc->len = p->dlen;
250 desc->flags |= VRING_DESC_F_NEXT;
251
252 /* set up the descriptor pointing to the buffer */
253 desc = virtio_desc_index_to_desc(vdev, RING_TX, desc->next);
254 desc->addr = pktbuf_data_phys(p2);
255 desc->len = p2->dlen;
256 desc->flags = 0;
257
258 /* submit the transfer */
259 virtio_submit_chain(vdev, RING_TX, i);
260
261 /* kick it off */
262 virtio_kick(vdev, RING_TX);
263
264 spin_unlock_irqrestore(&ndev->lock, state);
265
266 return NO_ERROR;
267 }
268
269 /* variant of the above function that copies the buffer into a pktbuf before sending */
virtio_net_queue_tx(struct virtio_net_dev * ndev,const void * buf,size_t len)270 static status_t virtio_net_queue_tx(struct virtio_net_dev *ndev, const void *buf, size_t len)
271 {
272 DEBUG_ASSERT(ndev);
273 DEBUG_ASSERT(buf);
274
275 pktbuf_t *p = pktbuf_alloc();
276 if (!p)
277 return ERR_NO_MEMORY;
278
279 /* copy the outgoing packet into the pktbuf */
280 p->data = p->buffer;
281 p->dlen = len;
282 memcpy(p->data, buf, len);
283
284 /* call through to the variant of the function that takes a pre-populated pktbuf */
285 status_t err = virtio_net_queue_tx_pktbuf(ndev, p);
286 if (err < 0) {
287 pktbuf_free(p, true);
288 }
289
290 return err;
291 }
292
virtio_net_queue_rx(struct virtio_net_dev * ndev,pktbuf_t * p)293 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p)
294 {
295 struct virtio_device *vdev = ndev->dev;
296
297 DEBUG_ASSERT(ndev);
298 DEBUG_ASSERT(p);
299
300 /* point our header to the base of the pktbuf */
301 p->data = p->buffer;
302 struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)p->data;
303 memset(hdr, 0, sizeof(struct virtio_net_hdr) - 2);
304
305 p->dlen = sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS;
306
307 spin_lock_saved_state_t state;
308 spin_lock_irqsave(&ndev->lock, state);
309
310 /* allocate a chain of descriptors for our transfer */
311 uint16_t i;
312 struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_RX, 1, &i);
313 DEBUG_ASSERT(desc); /* shouldn't be possible not to have a descriptor ready */
314
315 /* save a pointer to our pktbufs for the irq handler to use */
316 DEBUG_ASSERT(ndev->pending_rx_packet[i] == NULL);
317 ndev->pending_rx_packet[i] = p;
318
319 /* set up the descriptor pointing to the header */
320 desc->addr = pktbuf_data_phys(p);
321 desc->len = p->dlen;
322 desc->flags = VRING_DESC_F_WRITE;
323
324 /* submit the transfer */
325 virtio_submit_chain(vdev, RING_RX, i);
326
327 /* kick it off */
328 virtio_kick(vdev, RING_RX);
329
330 spin_unlock_irqrestore(&ndev->lock, state);
331
332 return NO_ERROR;
333 }
334
virtio_net_irq_driver_callback(struct virtio_device * dev,uint ring,const struct vring_used_elem * e)335 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e)
336 {
337 struct virtio_net_dev *ndev = (struct virtio_net_dev *)dev->priv;
338
339 LTRACEF("dev %p, ring %u, e %p, id %u, len %u\n", dev, ring, e, e->id, e->len);
340
341 spin_lock(&ndev->lock);
342
343 /* parse our descriptor chain, add back to the free queue */
344 uint16_t i = e->id;
345 for (;;) {
346 int next;
347 struct vring_desc *desc = virtio_desc_index_to_desc(dev, ring, i);
348
349 if (desc->flags & VRING_DESC_F_NEXT) {
350 next = desc->next;
351 } else {
352 /* end of chain */
353 next = -1;
354 }
355
356 virtio_free_desc(dev, ring, i);
357
358 if (ring == RING_RX) {
359 /* put the freed rx buffer in a queue */
360 pktbuf_t *p = ndev->pending_rx_packet[i];
361 ndev->pending_rx_packet[i] = NULL;
362
363 DEBUG_ASSERT(p);
364 LTRACEF("rx pktbuf %p filled\n", p);
365
366 /* trim the pktbuf according to the written length in the used element descriptor */
367 if (e->len > (sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS)) {
368 TRACEF("bad used len on RX %u\n", e->len);
369 p->dlen = 0;
370 } else {
371 p->dlen = e->len;
372 }
373
374 list_add_tail(&ndev->completed_rx_queue, &p->list);
375 } else { // ring == RING_TX
376 /* free the pktbuf associated with the tx packet we just consumed */
377 pktbuf_t *p = ndev->pending_tx_packet[i];
378 ndev->pending_tx_packet[i] = NULL;
379 ndev->tx_pending_count--;
380
381 DEBUG_ASSERT(p);
382 LTRACEF("freeing pktbuf %p\n", p);
383
384 pktbuf_free(p, false);
385 }
386
387 if (next < 0)
388 break;
389 i = next;
390 }
391
392 spin_unlock(&ndev->lock);
393
394 /* if rx ring, signal our event */
395 if (ring == 0) {
396 event_signal(&ndev->rx_event, false);
397 }
398
399 return INT_RESCHEDULE;
400 }
401
virtio_net_rx_worker(void * arg)402 static int virtio_net_rx_worker(void *arg)
403 {
404 struct virtio_net_dev *ndev = (struct virtio_net_dev *)arg;
405
406 for (;;) {
407 event_wait(&ndev->rx_event);
408
409 /* pull some packets from the received queue */
410 for (;;) {
411 spin_lock_saved_state_t state;
412 spin_lock_irqsave(&ndev->lock, state);
413
414 pktbuf_t *p = list_remove_head_type(&ndev->completed_rx_queue, pktbuf_t, list);
415
416 spin_unlock_irqrestore(&ndev->lock, state);
417
418 if (!p)
419 break; /* nothing left in the queue, go back to waiting */
420
421 LTRACEF("got packet len %u\n", p->dlen);
422
423 /* process our packet */
424 struct virtio_net_hdr *hdr = pktbuf_consume(p, sizeof(struct virtio_net_hdr) - 2);
425 if (hdr) {
426 /* call up into the stack */
427 minip_rx_driver_callback(p);
428 }
429
430 /* requeue the pktbuf in the rx queue */
431 virtio_net_queue_rx(ndev, p);
432 }
433 }
434 return 0;
435 }
436
virtio_net_found(void)437 int virtio_net_found(void)
438 {
439 return the_ndev ? 1 : 0;
440 }
441
virtio_net_get_mac_addr(uint8_t mac_addr[6])442 status_t virtio_net_get_mac_addr(uint8_t mac_addr[6])
443 {
444 if (!the_ndev)
445 return ERR_NOT_FOUND;
446
447 memcpy(mac_addr, the_ndev->config->mac, 6);
448
449 return NO_ERROR;
450 }
451
virtio_net_send_minip_pkt(pktbuf_t * p)452 status_t virtio_net_send_minip_pkt(pktbuf_t *p)
453 {
454 LTRACEF("p %p, dlen %u, flags 0x%x\n", p, p->dlen, p->flags);
455
456 DEBUG_ASSERT(p && p->dlen);
457
458 if ((p->flags & PKTBUF_FLAG_EOF) == 0) {
459 /* can't handle multi part packets yet */
460 PANIC_UNIMPLEMENTED;
461
462 return ERR_NOT_IMPLEMENTED;
463 }
464
465 /* hand the pktbuf off to the nic, it owns the pktbuf from now on out unless it fails */
466 status_t err = virtio_net_queue_tx_pktbuf(the_ndev, p);
467 if (err < 0) {
468 pktbuf_free(p, true);
469 }
470
471 return err;
472 }
473
474