1 // SPDX-License-Identifier: GPL-2.0-only
8  * DOC: cpu map
9  * The 'cpumap' is primarily used as a backend map for XDP BPF helper
13  * this map type redirects raw XDP frames to another CPU.  The remote
14  * CPU will do SKB-allocation and call the normal network stack.
20  * basically allows for 10G wirespeed pre-filtering via bpf.
39 /* General idea: XDP packets getting XDP redirected to another CPU,
40  * will maximum be stored/queued for one driver ->poll() call.  It is
42  * same CPU.  Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
46 #define CPU_MAP_BULK_SIZE 8  /* 8 == one cacheline on 64-bit archs */
57 /* Struct for every remote "destination" CPU in map */
59 	u32 cpu;    /* kthread CPU and map index */  member
60 	int map_id; /* Back reference to map */
62 	/* XDP can run multiple RX-ring queues, need __percpu enqueue store */
65 	/* Queue with potential multi-producers, and single-consumer kthread */
77 	struct bpf_map map;  member
78 	/* Below members specific for map type */
84 	u32 value_size = attr->value_size;  in cpu_map_alloc()
88 	if (attr->max_entries == 0 || attr->key_size != 4 ||  in cpu_map_alloc()
91 	    attr->map_flags & ~BPF_F_NUMA_NODE)  in cpu_map_alloc()
92 		return ERR_PTR(-EINVAL);  in cpu_map_alloc()
94 	/* Pre-limit array size based on NR_CPUS, not final CPU check */  in cpu_map_alloc()
95 	if (attr->max_entries > NR_CPUS)  in cpu_map_alloc()
96 		return ERR_PTR(-E2BIG);  in cpu_map_alloc()
100 		return ERR_PTR(-ENOMEM);  in cpu_map_alloc()
102 	bpf_map_init_from_attr(&cmap->map, attr);  in cpu_map_alloc()
105 	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *  in cpu_map_alloc()
107 					   cmap->map.numa_node);  in cpu_map_alloc()
108 	if (!cmap->cpu_map) {  in cpu_map_alloc()
110 		return ERR_PTR(-ENOMEM);  in cpu_map_alloc()
113 	return &cmap->map;  in cpu_map_alloc()
118 	/* The tear-down procedure should have made sure that queue is  in __cpu_map_ring_cleanup()
119 	 * empty.  See __cpu_map_entry_replace() and work-queue  in __cpu_map_ring_cleanup()
146 		act = bpf_prog_run_generic_xdp(skb, &xdp, rcpu->prog);  in cpu_map_bpf_prog_run_skb()
152 			err = xdp_do_generic_redirect(skb->dev, skb, &xdp,  in cpu_map_bpf_prog_run_skb()
153 						      rcpu->prog);  in cpu_map_bpf_prog_run_skb()
156 				stats->drop++;  in cpu_map_bpf_prog_run_skb()
158 				stats->redirect++;  in cpu_map_bpf_prog_run_skb()
162 			bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);  in cpu_map_bpf_prog_run_skb()
165 			trace_xdp_exception(skb->dev, rcpu->prog, act);  in cpu_map_bpf_prog_run_skb()
170 			stats->drop++;  in cpu_map_bpf_prog_run_skb()
192 		rxq.dev = xdpf->dev_rx;  in cpu_map_bpf_prog_run_xdp()
193 		rxq.mem.type = xdpf->mem_type;  in cpu_map_bpf_prog_run_xdp()
198 		act = bpf_prog_run_xdp(rcpu->prog, &xdp);  in cpu_map_bpf_prog_run_xdp()
204 				stats->drop++;  in cpu_map_bpf_prog_run_xdp()
207 				stats->pass++;  in cpu_map_bpf_prog_run_xdp()
211 			err = xdp_do_redirect(xdpf->dev_rx, &xdp,  in cpu_map_bpf_prog_run_xdp()
212 					      rcpu->prog);  in cpu_map_bpf_prog_run_xdp()
215 				stats->drop++;  in cpu_map_bpf_prog_run_xdp()
217 				stats->redirect++;  in cpu_map_bpf_prog_run_xdp()
221 			bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);  in cpu_map_bpf_prog_run_xdp()
225 			stats->drop++;  in cpu_map_bpf_prog_run_xdp()
244 	if (!rcpu->prog)  in cpu_map_bpf_prog_run()
252 	if (stats->redirect)  in cpu_map_bpf_prog_run()
269 	complete(&rcpu->kthread_running);  in cpu_map_kthread_run()
273 	 * from map, thus no new packets can enter. Remaining in-flight  in cpu_map_kthread_run()
274 	 * per CPU stored packets are flushed to this queue.  Wait honoring  in cpu_map_kthread_run()
277 	while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {  in cpu_map_kthread_run()
286 		/* Release CPU reschedule checks */  in cpu_map_kthread_run()
287 		if (__ptr_ring_empty(rcpu->queue)) {  in cpu_map_kthread_run()
289 			/* Recheck to avoid lost wake-up */  in cpu_map_kthread_run()
290 			if (__ptr_ring_empty(rcpu->queue)) {  in cpu_map_kthread_run()
304 		 * kthread CPU pinned. Lockless access to ptr_ring  in cpu_map_kthread_run()
305 		 * consume side valid as no-resize allowed of queue.  in cpu_map_kthread_run()
307 		n = __ptr_ring_consume_batched(rcpu->queue, frames,  in cpu_map_kthread_run()
317 				list_add_tail(&skb->list, &list);  in cpu_map_kthread_run()
324 			/* Bring struct page memory area to curr CPU. Read by  in cpu_map_kthread_run()
331 		/* Support running another XDP prog on this CPU */  in cpu_map_kthread_run()
349 							 xdpf->dev_rx);  in cpu_map_kthread_run()
355 			list_add_tail(&skb->list, &list);  in cpu_map_kthread_run()
361 		trace_xdp_cpumap_kthread(rcpu->map_id, n, kmem_alloc_drops,  in cpu_map_kthread_run()
373 				      struct bpf_map *map, int fd)  in __cpu_map_load_bpf_program()  argument
381 	if (prog->expected_attach_type != BPF_XDP_CPUMAP ||  in __cpu_map_load_bpf_program()
382 	    !bpf_prog_map_compatible(map, prog)) {  in __cpu_map_load_bpf_program()
384 		return -EINVAL;  in __cpu_map_load_bpf_program()
387 	rcpu->value.bpf_prog.id = prog->aux->id;  in __cpu_map_load_bpf_program()
388 	rcpu->prog = prog;  in __cpu_map_load_bpf_program()
394 __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,  in __cpu_map_entry_alloc()  argument
395 		      u32 cpu)  in __cpu_map_entry_alloc()  argument
397 	int numa, err, i, fd = value->bpf_prog.fd;  in __cpu_map_entry_alloc()
402 	/* Have map->numa_node, but choose node of redirect target CPU */  in __cpu_map_entry_alloc()
403 	numa = cpu_to_node(cpu);  in __cpu_map_entry_alloc()
405 	rcpu = bpf_map_kmalloc_node(map, sizeof(*rcpu), gfp | __GFP_ZERO, numa);  in __cpu_map_entry_alloc()
410 	rcpu->bulkq = bpf_map_alloc_percpu(map, sizeof(*rcpu->bulkq),  in __cpu_map_entry_alloc()
412 	if (!rcpu->bulkq)  in __cpu_map_entry_alloc()
416 		bq = per_cpu_ptr(rcpu->bulkq, i);  in __cpu_map_entry_alloc()
417 		bq->obj = rcpu;  in __cpu_map_entry_alloc()
421 	rcpu->queue = bpf_map_kmalloc_node(map, sizeof(*rcpu->queue), gfp,  in __cpu_map_entry_alloc()
423 	if (!rcpu->queue)  in __cpu_map_entry_alloc()
426 	err = ptr_ring_init(rcpu->queue, value->qsize, gfp);  in __cpu_map_entry_alloc()
430 	rcpu->cpu    = cpu;  in __cpu_map_entry_alloc()
431 	rcpu->map_id = map->id;  in __cpu_map_entry_alloc()
432 	rcpu->value.qsize  = value->qsize;  in __cpu_map_entry_alloc()
434 	if (fd > 0 && __cpu_map_load_bpf_program(rcpu, map, fd))  in __cpu_map_entry_alloc()
438 	init_completion(&rcpu->kthread_running);  in __cpu_map_entry_alloc()
439 	rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,  in __cpu_map_entry_alloc()
440 					       "cpumap/%d/map:%d", cpu,  in __cpu_map_entry_alloc()
441 					       map->id);  in __cpu_map_entry_alloc()
442 	if (IS_ERR(rcpu->kthread))  in __cpu_map_entry_alloc()
445 	/* Make sure kthread runs on a single CPU */  in __cpu_map_entry_alloc()
446 	kthread_bind(rcpu->kthread, cpu);  in __cpu_map_entry_alloc()
447 	wake_up_process(rcpu->kthread);  in __cpu_map_entry_alloc()
453 	wait_for_completion(&rcpu->kthread_running);  in __cpu_map_entry_alloc()
458 	if (rcpu->prog)  in __cpu_map_entry_alloc()
459 		bpf_prog_put(rcpu->prog);  in __cpu_map_entry_alloc()
461 	ptr_ring_cleanup(rcpu->queue, NULL);  in __cpu_map_entry_alloc()
463 	kfree(rcpu->queue);  in __cpu_map_entry_alloc()
465 	free_percpu(rcpu->bulkq);  in __cpu_map_entry_alloc()
475 	/* This cpu_map_entry have been disconnected from map and one  in __cpu_map_entry_free()
476 	 * RCU grace-period have elapsed. Thus, XDP cannot queue any  in __cpu_map_entry_free()
486 	kthread_stop(rcpu->kthread);  in __cpu_map_entry_free()
488 	if (rcpu->prog)  in __cpu_map_entry_free()
489 		bpf_prog_put(rcpu->prog);  in __cpu_map_entry_free()
491 	__cpu_map_ring_cleanup(rcpu->queue);  in __cpu_map_entry_free()
492 	ptr_ring_cleanup(rcpu->queue, NULL);  in __cpu_map_entry_free()
493 	kfree(rcpu->queue);  in __cpu_map_entry_free()
494 	free_percpu(rcpu->bulkq);  in __cpu_map_entry_free()
511 	old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu)));  in __cpu_map_entry_replace()
513 		INIT_RCU_WORK(&old_rcpu->free_work, __cpu_map_entry_free);  in __cpu_map_entry_replace()
514 		queue_rcu_work(system_wq, &old_rcpu->free_work);  in __cpu_map_entry_replace()
518 static long cpu_map_delete_elem(struct bpf_map *map, void *key)  in cpu_map_delete_elem()  argument
520 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_delete_elem()
523 	if (key_cpu >= map->max_entries)  in cpu_map_delete_elem()
524 		return -EINVAL;  in cpu_map_delete_elem()
531 static long cpu_map_update_elem(struct bpf_map *map, void *key, void *value,  in cpu_map_update_elem()  argument
534 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_update_elem()
537 	/* Array index key correspond to CPU number */  in cpu_map_update_elem()
540 	memcpy(&cpumap_value, value, map->value_size);  in cpu_map_update_elem()
543 		return -EINVAL;  in cpu_map_update_elem()
544 	if (unlikely(key_cpu >= cmap->map.max_entries))  in cpu_map_update_elem()
545 		return -E2BIG;  in cpu_map_update_elem()
547 		return -EEXIST;  in cpu_map_update_elem()
549 		return -EOVERFLOW;  in cpu_map_update_elem()
551 	/* Make sure CPU is a valid possible cpu */  in cpu_map_update_elem()
553 		return -ENODEV;  in cpu_map_update_elem()
558 		/* Updating qsize cause re-allocation of bpf_cpu_map_entry */  in cpu_map_update_elem()
559 		rcpu = __cpu_map_entry_alloc(map, &cpumap_value, key_cpu);  in cpu_map_update_elem()
561 			return -ENOMEM;  in cpu_map_update_elem()
569 static void cpu_map_free(struct bpf_map *map)  in cpu_map_free()  argument
571 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_free()
574 	/* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,  in cpu_map_free()
575 	 * so the bpf programs (can be more than one that used this map) were  in cpu_map_free()
578 	 * guarantees no further "XDP/bpf-side" reads against  in cpu_map_free()
579 	 * bpf_cpu_map->cpu_map, but also ensure pending flush operations  in cpu_map_free()
587 	for (i = 0; i < cmap->map.max_entries; i++) {  in cpu_map_free()
590 		rcpu = rcu_dereference_raw(cmap->cpu_map[i]);  in cpu_map_free()
595 		__cpu_map_entry_free(&rcpu->free_work.work);  in cpu_map_free()
597 	bpf_map_area_free(cmap->cpu_map);  in cpu_map_free()
605 static void *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)  in __cpu_map_lookup_elem()  argument
607 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in __cpu_map_lookup_elem()
610 	if (key >= map->max_entries)  in __cpu_map_lookup_elem()
613 	rcpu = rcu_dereference_check(cmap->cpu_map[key],  in __cpu_map_lookup_elem()
618 static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)  in cpu_map_lookup_elem()  argument
621 		__cpu_map_lookup_elem(map, *(u32 *)key);  in cpu_map_lookup_elem()
623 	return rcpu ? &rcpu->value : NULL;  in cpu_map_lookup_elem()
626 static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in cpu_map_get_next_key()  argument
628 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_get_next_key()
632 	if (index >= cmap->map.max_entries) {  in cpu_map_get_next_key()
637 	if (index == cmap->map.max_entries - 1)  in cpu_map_get_next_key()
638 		return -ENOENT;  in cpu_map_get_next_key()
643 static long cpu_map_redirect(struct bpf_map *map, u64 index, u64 flags)  in cpu_map_redirect()  argument
645 	return __bpf_xdp_redirect_map(map, index, flags, 0,  in cpu_map_redirect()
649 static u64 cpu_map_mem_usage(const struct bpf_map *map)  in cpu_map_mem_usage()  argument
654 	usage += (u64)map->max_entries * sizeof(struct bpf_cpu_map_entry *);  in cpu_map_mem_usage()
675 	struct bpf_cpu_map_entry *rcpu = bq->obj;  in bq_flush_to_queue()
677 	const int to_cpu = rcpu->cpu;  in bq_flush_to_queue()
681 	if (unlikely(!bq->count))  in bq_flush_to_queue()
684 	q = rcpu->queue;  in bq_flush_to_queue()
685 	spin_lock(&q->producer_lock);  in bq_flush_to_queue()
687 	for (i = 0; i < bq->count; i++) {  in bq_flush_to_queue()
688 		struct xdp_frame *xdpf = bq->q[i];  in bq_flush_to_queue()
698 	bq->count = 0;  in bq_flush_to_queue()
699 	spin_unlock(&q->producer_lock);  in bq_flush_to_queue()
701 	__list_del_clearprev(&bq->flush_node);  in bq_flush_to_queue()
704 	trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);  in bq_flush_to_queue()
707 /* Runs under RCU-read-side, plus in softirq under NAPI protection.
712 	struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);  in bq_enqueue()
714 	if (unlikely(bq->count == CPU_MAP_BULK_SIZE))  in bq_enqueue()
719 	 * (e.g. ixgbe) recycle tricks based on page-refcnt.  in bq_enqueue()
722 	 * with another CPU on page-refcnt and remaining driver code).  in bq_enqueue()
724 	 * operation, when completing napi->poll call.  in bq_enqueue()
726 	bq->q[bq->count++] = xdpf;  in bq_enqueue()
728 	if (!bq->flush_node.prev) {  in bq_enqueue()
731 		list_add(&bq->flush_node, flush_list);  in bq_enqueue()
738 	/* Info needed when constructing SKB on remote CPU */  in cpu_map_enqueue()
739 	xdpf->dev_rx = dev_rx;  in cpu_map_enqueue()
750 	__skb_pull(skb, skb->mac_len);  in cpu_map_generic_redirect()
754 	ret = ptr_ring_produce(rcpu->queue, skb);  in cpu_map_generic_redirect()
758 	wake_up_process(rcpu->kthread);  in cpu_map_generic_redirect()
760 	trace_xdp_cpumap_enqueue(rcpu->map_id, !ret, !!ret, rcpu->cpu);  in cpu_map_generic_redirect()
772 		wake_up_process(bq->obj->kthread);  in __cpu_map_flush()