3  * drivers/net/team/team.c - Network team device driver
32 #define DRV_NAME "team"
65 static int team_port_set_team_dev_addr(struct team *team,  in team_port_set_team_dev_addr()  argument
68 	return __set_port_dev_addr(port->dev, team->dev->dev_addr);  in team_port_set_team_dev_addr()
71 int team_modeop_port_enter(struct team *team, struct team_port *port)  in team_modeop_port_enter()  argument
73 	return team_port_set_team_dev_addr(team, port);  in team_modeop_port_enter()
77 void team_modeop_port_change_dev_addr(struct team *team,  in team_modeop_port_change_dev_addr()  argument
80 	team_port_set_team_dev_addr(team, port);  in team_modeop_port_change_dev_addr()
118 static struct team_option *__team_find_option(struct team *team,  in __team_find_option()  argument
123 	list_for_each_entry(option, &team->option_list, list) {  in __team_find_option()
136 static void __team_option_inst_del_option(struct team *team,  in __team_option_inst_del_option()  argument
141 	list_for_each_entry_safe(opt_inst, tmp, &team->option_inst_list, list) {  in __team_option_inst_del_option()
147 static int __team_option_inst_add(struct team *team, struct team_option *option,  in __team_option_inst_add()  argument
167 		list_add_tail(&opt_inst->list, &team->option_inst_list);  in __team_option_inst_add()
169 			option->init(team, &opt_inst->info);  in __team_option_inst_add()
175 static int __team_option_inst_add_option(struct team *team,  in __team_option_inst_add_option()  argument
181 		err = __team_option_inst_add(team, option, NULL);  in __team_option_inst_add_option()
188 	__team_option_inst_del_option(team, option);  in __team_option_inst_add_option()
192 static void __team_option_inst_mark_removed_option(struct team *team,  in __team_option_inst_mark_removed_option()  argument
197 	list_for_each_entry(opt_inst, &team->option_inst_list, list) {  in __team_option_inst_mark_removed_option()
205 static void __team_option_inst_del_port(struct team *team,  in __team_option_inst_del_port()  argument
210 	list_for_each_entry_safe(opt_inst, tmp, &team->option_inst_list, list) {  in __team_option_inst_del_port()
217 static int __team_option_inst_add_port(struct team *team,  in __team_option_inst_add_port()  argument
223 	list_for_each_entry(option, &team->option_list, list) {  in __team_option_inst_add_port()
226 		err = __team_option_inst_add(team, option, port);  in __team_option_inst_add_port()
233 	__team_option_inst_del_port(team, port);  in __team_option_inst_add_port()
237 static void __team_option_inst_mark_removed_port(struct team *team,  in __team_option_inst_mark_removed_port()  argument
242 	list_for_each_entry(opt_inst, &team->option_inst_list, list) {  in __team_option_inst_mark_removed_port()
250 static int __team_options_register(struct team *team,  in __team_options_register()  argument
263 		if (__team_find_option(team, option->name)) {  in __team_options_register()
275 		err = __team_option_inst_add_option(team, dst_opts[i]);  in __team_options_register()
278 		list_add_tail(&dst_opts[i]->list, &team->option_list);  in __team_options_register()
286 		__team_option_inst_del_option(team, dst_opts[i]);  in __team_options_register()
299 static void __team_options_mark_removed(struct team *team,  in __team_options_mark_removed()  argument
308 		del_opt = __team_find_option(team, option->name);  in __team_options_mark_removed()
310 			__team_option_inst_mark_removed_option(team, del_opt);  in __team_options_mark_removed()
314 static void __team_options_unregister(struct team *team,  in __team_options_unregister()  argument
323 		del_opt = __team_find_option(team, option->name);  in __team_options_unregister()
325 			__team_option_inst_del_option(team, del_opt);  in __team_options_unregister()
332 static void __team_options_change_check(struct team *team);
334 int team_options_register(struct team *team,  in team_options_register()  argument
340 	err = __team_options_register(team, option, option_count);  in team_options_register()
343 	__team_options_change_check(team);  in team_options_register()
348 void team_options_unregister(struct team *team,  in team_options_unregister()  argument
352 	__team_options_mark_removed(team, option, option_count);  in team_options_unregister()
353 	__team_options_change_check(team);  in team_options_unregister()
354 	__team_options_unregister(team, option, option_count);  in team_options_unregister()
358 static int team_option_get(struct team *team,  in team_option_get()  argument
365 	opt_inst->option->getter(team, ctx);  in team_option_get()
369 static int team_option_set(struct team *team,  in team_option_set()  argument
375 	return opt_inst->option->setter(team, ctx);  in team_option_set()
387 void team_options_change_check(struct team *team)  in team_options_change_check()  argument
389 	__team_options_change_check(team);  in team_options_change_check()
480 		request_module("team-mode-%s", kind);  in team_mode_get()
500 static bool team_dummy_transmit(struct team *team, struct sk_buff *skb)  in team_dummy_transmit()  argument
506 static rx_handler_result_t team_dummy_receive(struct team *team,  in team_dummy_receive()  argument
517 static bool team_is_mode_set(struct team *team)  in team_is_mode_set()  argument
519 	return team->mode != &__team_no_mode;  in team_is_mode_set()
522 static void team_set_no_mode(struct team *team)  in team_set_no_mode()  argument
524 	team->user_carrier_enabled = false;  in team_set_no_mode()
525 	team->mode = &__team_no_mode;  in team_set_no_mode()
528 static void team_adjust_ops(struct team *team)  in team_adjust_ops()  argument
535 	if (!team->en_port_count || !team_is_mode_set(team) ||  in team_adjust_ops()
536 	    !team->mode->ops->transmit)  in team_adjust_ops()
537 		team->ops.transmit = team_dummy_transmit;  in team_adjust_ops()
539 		team->ops.transmit = team->mode->ops->transmit;  in team_adjust_ops()
541 	if (!team->en_port_count || !team_is_mode_set(team) ||  in team_adjust_ops()
542 	    !team->mode->ops->receive)  in team_adjust_ops()
543 		team->ops.receive = team_dummy_receive;  in team_adjust_ops()
545 		team->ops.receive = team->mode->ops->receive;  in team_adjust_ops()
553 static int __team_change_mode(struct team *team,  in __team_change_mode()  argument
557 	if (team_is_mode_set(team)) {  in __team_change_mode()
558 		void (*exit_op)(struct team *team) = team->ops.exit;  in __team_change_mode()
561 		memset(&team->ops, 0, sizeof(struct team_mode_ops));  in __team_change_mode()
562 		team_adjust_ops(team);  in __team_change_mode()
565 			exit_op(team);  in __team_change_mode()
566 		team_mode_put(team->mode);  in __team_change_mode()
567 		team_set_no_mode(team);  in __team_change_mode()
569 		memset(&team->mode_priv, 0,  in __team_change_mode()
570 		       sizeof(struct team) - offsetof(struct team, mode_priv));  in __team_change_mode()
579 		err = new_mode->ops->init(team);  in __team_change_mode()
584 	team->mode = new_mode;  in __team_change_mode()
585 	memcpy(&team->ops, new_mode->ops, sizeof(struct team_mode_ops));  in __team_change_mode()
586 	team_adjust_ops(team);  in __team_change_mode()
591 static int team_change_mode(struct team *team, const char *kind)  in team_change_mode()  argument
594 	struct net_device *dev = team->dev;  in team_change_mode()
597 	if (!list_empty(&team->port_list)) {  in team_change_mode()
602 	if (team_is_mode_set(team) && strcmp(team->mode->kind, kind) == 0) {  in team_change_mode()
603 		netdev_err(dev, "Unable to change to the same mode the team is in\n");  in team_change_mode()
613 	err = __team_change_mode(team, new_mode);  in team_change_mode()
631 	struct team *team;  in team_notify_peers_work()  local
634 	team = container_of(work, struct team, notify_peers.dw.work);  in team_notify_peers_work()
637 		schedule_delayed_work(&team->notify_peers.dw, 0);  in team_notify_peers_work()
640 	val = atomic_dec_if_positive(&team->notify_peers.count_pending);  in team_notify_peers_work()
645 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, team->dev);  in team_notify_peers_work()
648 		schedule_delayed_work(&team->notify_peers.dw,  in team_notify_peers_work()
649 				      msecs_to_jiffies(team->notify_peers.interval));  in team_notify_peers_work()
652 static void team_notify_peers(struct team *team)  in team_notify_peers()  argument
654 	if (!team->notify_peers.count || !netif_running(team->dev))  in team_notify_peers()
656 	atomic_add(team->notify_peers.count, &team->notify_peers.count_pending);  in team_notify_peers()
657 	schedule_delayed_work(&team->notify_peers.dw, 0);  in team_notify_peers()
660 static void team_notify_peers_init(struct team *team)  in team_notify_peers_init()  argument
662 	INIT_DELAYED_WORK(&team->notify_peers.dw, team_notify_peers_work);  in team_notify_peers_init()
665 static void team_notify_peers_fini(struct team *team)  in team_notify_peers_fini()  argument
667 	cancel_delayed_work_sync(&team->notify_peers.dw);  in team_notify_peers_fini()
677 	struct team *team;  in team_mcast_rejoin_work()  local
680 	team = container_of(work, struct team, mcast_rejoin.dw.work);  in team_mcast_rejoin_work()
683 		schedule_delayed_work(&team->mcast_rejoin.dw, 0);  in team_mcast_rejoin_work()
686 	val = atomic_dec_if_positive(&team->mcast_rejoin.count_pending);  in team_mcast_rejoin_work()
691 	call_netdevice_notifiers(NETDEV_RESEND_IGMP, team->dev);  in team_mcast_rejoin_work()
694 		schedule_delayed_work(&team->mcast_rejoin.dw,  in team_mcast_rejoin_work()
695 				      msecs_to_jiffies(team->mcast_rejoin.interval));  in team_mcast_rejoin_work()
698 static void team_mcast_rejoin(struct team *team)  in team_mcast_rejoin()  argument
700 	if (!team->mcast_rejoin.count || !netif_running(team->dev))  in team_mcast_rejoin()
702 	atomic_add(team->mcast_rejoin.count, &team->mcast_rejoin.count_pending);  in team_mcast_rejoin()
703 	schedule_delayed_work(&team->mcast_rejoin.dw, 0);  in team_mcast_rejoin()
706 static void team_mcast_rejoin_init(struct team *team)  in team_mcast_rejoin_init()  argument
708 	INIT_DELAYED_WORK(&team->mcast_rejoin.dw, team_mcast_rejoin_work);  in team_mcast_rejoin_init()
711 static void team_mcast_rejoin_fini(struct team *team)  in team_mcast_rejoin_fini()  argument
713 	cancel_delayed_work_sync(&team->mcast_rejoin.dw);  in team_mcast_rejoin_fini()
726 	struct team *team;  in team_handle_frame()  local
736 	team = port->team;  in team_handle_frame()
746 		res = team->ops.receive(team, port, skb);  in team_handle_frame()
751 		pcpu_stats = this_cpu_ptr(team->pcpu_stats);  in team_handle_frame()
759 		skb->dev = team->dev;  in team_handle_frame()
761 		this_cpu_inc(team->pcpu_stats->rx_nohandler);  in team_handle_frame()
763 		this_cpu_inc(team->pcpu_stats->rx_dropped);  in team_handle_frame()
774 static int team_queue_override_init(struct team *team)  in team_queue_override_init()  argument
777 	unsigned int queue_cnt = team->dev->num_tx_queues - 1;  in team_queue_override_init()
786 	team->qom_lists = listarr;  in team_queue_override_init()
792 static void team_queue_override_fini(struct team *team)  in team_queue_override_fini()  argument
794 	kfree(team->qom_lists);  in team_queue_override_fini()
797 static struct list_head *__team_get_qom_list(struct team *team, u16 queue_id)  in __team_get_qom_list()  argument
799 	return &team->qom_lists[queue_id - 1];  in __team_get_qom_list()
805 static bool team_queue_override_transmit(struct team *team, struct sk_buff *skb)  in team_queue_override_transmit()  argument
810 	if (!team->queue_override_enabled || !skb->queue_mapping)  in team_queue_override_transmit()
812 	qom_list = __team_get_qom_list(team, skb->queue_mapping);  in team_queue_override_transmit()
814 		if (!team_dev_queue_xmit(team, port, skb))  in team_queue_override_transmit()
820 static void __team_queue_override_port_del(struct team *team,  in __team_queue_override_port_del()  argument
840 static void __team_queue_override_port_add(struct team *team,  in __team_queue_override_port_add()  argument
849 	qom_list = __team_get_qom_list(team, port->queue_id);  in __team_queue_override_port_add()
859 static void __team_queue_override_enabled_check(struct team *team)  in __team_queue_override_enabled_check()  argument
864 	list_for_each_entry(port, &team->port_list, list) {  in __team_queue_override_enabled_check()
870 	if (enabled == team->queue_override_enabled)  in __team_queue_override_enabled_check()
872 	netdev_dbg(team->dev, "%s queue override\n",  in __team_queue_override_enabled_check()
874 	team->queue_override_enabled = enabled;  in __team_queue_override_enabled_check()
877 static void team_queue_override_port_prio_changed(struct team *team,  in team_queue_override_port_prio_changed()  argument
882 	__team_queue_override_port_del(team, port);  in team_queue_override_port_prio_changed()
883 	__team_queue_override_port_add(team, port);  in team_queue_override_port_prio_changed()
884 	__team_queue_override_enabled_check(team);  in team_queue_override_port_prio_changed()
887 static void team_queue_override_port_change_queue_id(struct team *team,  in team_queue_override_port_change_queue_id()  argument
892 		__team_queue_override_port_del(team, port);  in team_queue_override_port_change_queue_id()
894 		__team_queue_override_port_add(team, port);  in team_queue_override_port_change_queue_id()
895 		__team_queue_override_enabled_check(team);  in team_queue_override_port_change_queue_id()
901 static void team_queue_override_port_add(struct team *team,  in team_queue_override_port_add()  argument
904 	__team_queue_override_port_add(team, port);  in team_queue_override_port_add()
905 	__team_queue_override_enabled_check(team);  in team_queue_override_port_add()
908 static void team_queue_override_port_del(struct team *team,  in team_queue_override_port_del()  argument
911 	__team_queue_override_port_del(team, port);  in team_queue_override_port_del()
912 	__team_queue_override_enabled_check(team);  in team_queue_override_port_del()
920 static bool team_port_find(const struct team *team,  in team_port_find()  argument
925 	list_for_each_entry(cur, &team->port_list, list)  in team_port_find()
935  * by team->lock.
937 static void team_port_enable(struct team *team,  in team_port_enable()  argument
942 	port->index = team->en_port_count++;  in team_port_enable()
944 			   team_port_index_hash(team, port->index));  in team_port_enable()
945 	team_adjust_ops(team);  in team_port_enable()
946 	team_queue_override_port_add(team, port);  in team_port_enable()
947 	if (team->ops.port_enabled)  in team_port_enable()
948 		team->ops.port_enabled(team, port);  in team_port_enable()
949 	team_notify_peers(team);  in team_port_enable()
950 	team_mcast_rejoin(team);  in team_port_enable()
954 static void __reconstruct_port_hlist(struct team *team, int rm_index)  in __reconstruct_port_hlist()  argument
959 	for (i = rm_index + 1; i < team->en_port_count; i++) {  in __reconstruct_port_hlist()
960 		port = team_get_port_by_index(team, i);  in __reconstruct_port_hlist()
964 				   team_port_index_hash(team, port->index));  in __reconstruct_port_hlist()
968 static void team_port_disable(struct team *team,  in team_port_disable()  argument
973 	if (team->ops.port_disabled)  in team_port_disable()
974 		team->ops.port_disabled(team, port);  in team_port_disable()
976 	__reconstruct_port_hlist(team, port->index);  in team_port_disable()
978 	team->en_port_count--;  in team_port_disable()
979 	team_queue_override_port_del(team, port);  in team_port_disable()
980 	team_adjust_ops(team);  in team_port_disable()
992 static void __team_compute_features(struct team *team)  in __team_compute_features()  argument
1002 	if (list_empty(&team->port_list))  in __team_compute_features()
1008 	list_for_each_entry_rcu(port, &team->port_list, list) {  in __team_compute_features()
1024 	team->dev->vlan_features = vlan_features;  in __team_compute_features()
1025 	team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL |  in __team_compute_features()
1028 	team->dev->hard_header_len = max_hard_header_len;  in __team_compute_features()
1030 	team->dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;  in __team_compute_features()
1032 		team->dev->priv_flags |= IFF_XMIT_DST_RELEASE;  in __team_compute_features()
1035 static void team_compute_features(struct team *team)  in team_compute_features()  argument
1037 	__team_compute_features(team);  in team_compute_features()
1038 	netdev_change_features(team->dev);  in team_compute_features()
1041 static int team_port_enter(struct team *team, struct team_port *port)  in team_port_enter()  argument
1045 	dev_hold(team->dev);  in team_port_enter()
1046 	if (team->ops.port_enter) {  in team_port_enter()
1047 		err = team->ops.port_enter(team, port);  in team_port_enter()
1049 			netdev_err(team->dev, "Device %s failed to enter team mode\n",  in team_port_enter()
1058 	dev_put(team->dev);  in team_port_enter()
1063 static void team_port_leave(struct team *team, struct team_port *port)  in team_port_leave()  argument
1065 	if (team->ops.port_leave)  in team_port_leave()
1066 		team->ops.port_leave(team, port);  in team_port_leave()
1067 	dev_put(team->dev);  in team_port_leave()
1091 	if (!port->team->dev->npinfo)  in team_port_enable_netpoll()
1117 static int team_upper_dev_link(struct team *team, struct team_port *port,  in team_upper_dev_link()  argument
1123 	lag_upper_info.tx_type = team->mode->lag_tx_type;  in team_upper_dev_link()
1125 	err = netdev_master_upper_dev_link(port->dev, team->dev, NULL,  in team_upper_dev_link()
1133 static void team_upper_dev_unlink(struct team *team, struct team_port *port)  in team_upper_dev_unlink()  argument
1135 	netdev_upper_dev_unlink(port->dev, team->dev);  in team_upper_dev_unlink()
1143 static int team_port_add(struct team *team, struct net_device *port_dev,  in team_port_add()  argument
1146 	struct net_device *dev = team->dev;  in team_port_add()
1152 		NL_SET_ERR_MSG(extack, "Loopback device can't be added as a team port");  in team_port_add()
1153 		netdev_err(dev, "Device %s is loopback device. Loopback devices can't be added as a team port\n",  in team_port_add()
1159 		NL_SET_ERR_MSG(extack, "Device is already a port of a team device");  in team_port_add()
1161 				"of a team device\n", portname);  in team_port_add()
1166 		NL_SET_ERR_MSG(extack, "Cannot enslave team device to itself");  in team_port_add()
1167 		netdev_err(dev, "Cannot enslave team device to itself\n");  in team_port_add()
1172 		NL_SET_ERR_MSG(extack, "Device is already an upper device of the team interface");  in team_port_add()
1173 		netdev_err(dev, "Device %s is already an upper device of the team interface\n",  in team_port_add()
1179 		NL_SET_ERR_MSG(extack, "Device is already a lower device of the team interface");  in team_port_add()
1180 		netdev_err(dev, "Device %s is already a lower device of the team interface\n",  in team_port_add()
1187 		NL_SET_ERR_MSG(extack, "Device is VLAN challenged and team device has VLAN set up");  in team_port_add()
1188 		netdev_err(dev, "Device %s is VLAN challenged and team device has VLAN set up\n",  in team_port_add()
1198 		NL_SET_ERR_MSG(extack, "Device is up. Set it down before adding it as a team port");  in team_port_add()
1199 		netdev_err(dev, "Device %s is up. Set it down before adding it as a team port\n",  in team_port_add()
1204 	port = kzalloc(sizeof(struct team_port) + team->mode->port_priv_size,  in team_port_add()
1210 	port->team = team;  in team_port_add()
1222 	err = team_port_enter(team, port);  in team_port_add()
1224 		netdev_err(dev, "Device %s failed to enter team mode\n",  in team_port_add()
1261 	err = team_upper_dev_link(team, port, extack);  in team_port_add()
1268 	err = __team_option_inst_add_port(team, port);  in team_port_add()
1300 	list_add_tail_rcu(&port->list, &team->port_list);  in team_port_add()
1301 	team_port_enable(team, port);  in team_port_add()
1302 	__team_compute_features(team);  in team_port_add()
1304 	__team_options_change_check(team);  in team_port_add()
1311 	__team_option_inst_del_port(team, port);  in team_port_add()
1314 	team_upper_dev_unlink(team, port);  in team_port_add()
1329 	team_port_leave(team, port);  in team_port_add()
1343 static int team_port_del(struct team *team, struct net_device *port_dev)  in team_port_del()  argument
1345 	struct net_device *dev = team->dev;  in team_port_del()
1350 	if (!port || !team_port_find(team, port)) {  in team_port_del()
1351 		netdev_err(dev, "Device %s does not act as a port of this team\n",  in team_port_del()
1356 	team_port_disable(team, port);  in team_port_del()
1364 	team_upper_dev_unlink(team, port);  in team_port_del()
1373 	team_port_leave(team, port);  in team_port_del()
1375 	__team_option_inst_mark_removed_port(team, port);  in team_port_del()
1376 	__team_options_change_check(team);  in team_port_del()
1377 	__team_option_inst_del_port(team, port);  in team_port_del()
1384 	__team_compute_features(team);  in team_port_del()
1394 static void team_mode_option_get(struct team *team, struct team_gsetter_ctx *ctx)  in team_mode_option_get()  argument
1396 	ctx->data.str_val = team->mode->kind;  in team_mode_option_get()
1399 static int team_mode_option_set(struct team *team, struct team_gsetter_ctx *ctx)  in team_mode_option_set()  argument
1401 	return team_change_mode(team, ctx->data.str_val);  in team_mode_option_set()
1404 static void team_notify_peers_count_get(struct team *team,  in team_notify_peers_count_get()  argument
1407 	ctx->data.u32_val = team->notify_peers.count;  in team_notify_peers_count_get()
1410 static int team_notify_peers_count_set(struct team *team,  in team_notify_peers_count_set()  argument
1413 	team->notify_peers.count = ctx->data.u32_val;  in team_notify_peers_count_set()
1417 static void team_notify_peers_interval_get(struct team *team,  in team_notify_peers_interval_get()  argument
1420 	ctx->data.u32_val = team->notify_peers.interval;  in team_notify_peers_interval_get()
1423 static int team_notify_peers_interval_set(struct team *team,  in team_notify_peers_interval_set()  argument
1426 	team->notify_peers.interval = ctx->data.u32_val;  in team_notify_peers_interval_set()
1430 static void team_mcast_rejoin_count_get(struct team *team,  in team_mcast_rejoin_count_get()  argument
1433 	ctx->data.u32_val = team->mcast_rejoin.count;  in team_mcast_rejoin_count_get()
1436 static int team_mcast_rejoin_count_set(struct team *team,  in team_mcast_rejoin_count_set()  argument
1439 	team->mcast_rejoin.count = ctx->data.u32_val;  in team_mcast_rejoin_count_set()
1443 static void team_mcast_rejoin_interval_get(struct team *team,  in team_mcast_rejoin_interval_get()  argument
1446 	ctx->data.u32_val = team->mcast_rejoin.interval;  in team_mcast_rejoin_interval_get()
1449 static int team_mcast_rejoin_interval_set(struct team *team,  in team_mcast_rejoin_interval_set()  argument
1452 	team->mcast_rejoin.interval = ctx->data.u32_val;  in team_mcast_rejoin_interval_set()
1456 static void team_port_en_option_get(struct team *team,  in team_port_en_option_get()  argument
1464 static int team_port_en_option_set(struct team *team,  in team_port_en_option_set()  argument
1470 		team_port_enable(team, port);  in team_port_en_option_set()
1472 		team_port_disable(team, port);  in team_port_en_option_set()
1476 static void team_user_linkup_option_get(struct team *team,  in team_user_linkup_option_get()  argument
1484 static void __team_carrier_check(struct team *team);
1486 static int team_user_linkup_option_set(struct team *team,  in team_user_linkup_option_set()  argument
1493 	__team_carrier_check(port->team);  in team_user_linkup_option_set()
1497 static void team_user_linkup_en_option_get(struct team *team,  in team_user_linkup_en_option_get()  argument
1505 static int team_user_linkup_en_option_set(struct team *team,  in team_user_linkup_en_option_set()  argument
1512 	__team_carrier_check(port->team);  in team_user_linkup_en_option_set()
1516 static void team_priority_option_get(struct team *team,  in team_priority_option_get()  argument
1524 static int team_priority_option_set(struct team *team,  in team_priority_option_set()  argument
1533 	team_queue_override_port_prio_changed(team, port);  in team_priority_option_set()
1537 static void team_queue_id_option_get(struct team *team,  in team_queue_id_option_get()  argument
1545 static int team_queue_id_option_set(struct team *team,  in team_queue_id_option_set()  argument
1553 	if (new_queue_id >= team->dev->real_num_tx_queues)  in team_queue_id_option_set()
1555 	team_queue_override_port_change_queue_id(team, port, new_queue_id);  in team_queue_id_option_set()
1630 	struct team *team = netdev_priv(dev);  in team_init()  local
1634 	team->dev = dev;  in team_init()
1635 	team_set_no_mode(team);  in team_init()
1636 	team->notifier_ctx = false;  in team_init()
1638 	team->pcpu_stats = netdev_alloc_pcpu_stats(struct team_pcpu_stats);  in team_init()
1639 	if (!team->pcpu_stats)  in team_init()
1643 		INIT_HLIST_HEAD(&team->en_port_hlist[i]);  in team_init()
1644 	INIT_LIST_HEAD(&team->port_list);  in team_init()
1645 	err = team_queue_override_init(team);  in team_init()
1649 	team_adjust_ops(team);  in team_init()
1651 	INIT_LIST_HEAD(&team->option_list);  in team_init()
1652 	INIT_LIST_HEAD(&team->option_inst_list);  in team_init()
1654 	team_notify_peers_init(team);  in team_init()
1655 	team_mcast_rejoin_init(team);  in team_init()
1657 	err = team_options_register(team, team_options, ARRAY_SIZE(team_options));  in team_init()
1662 	lockdep_register_key(&team->team_lock_key);  in team_init()
1663 	__mutex_init(&team->lock, "team->team_lock_key", &team->team_lock_key);  in team_init()
1669 	team_mcast_rejoin_fini(team);  in team_init()
1670 	team_notify_peers_fini(team);  in team_init()
1671 	team_queue_override_fini(team);  in team_init()
1673 	free_percpu(team->pcpu_stats);  in team_init()
1680 	struct team *team = netdev_priv(dev);  in team_uninit()  local
1684 	mutex_lock(&team->lock);  in team_uninit()
1685 	list_for_each_entry_safe(port, tmp, &team->port_list, list)  in team_uninit()
1686 		team_port_del(team, port->dev);  in team_uninit()
1688 	__team_change_mode(team, NULL); /* cleanup */  in team_uninit()
1689 	__team_options_unregister(team, team_options, ARRAY_SIZE(team_options));  in team_uninit()
1690 	team_mcast_rejoin_fini(team);  in team_uninit()
1691 	team_notify_peers_fini(team);  in team_uninit()
1692 	team_queue_override_fini(team);  in team_uninit()
1693 	mutex_unlock(&team->lock);  in team_uninit()
1695 	lockdep_unregister_key(&team->team_lock_key);  in team_uninit()
1700 	struct team *team = netdev_priv(dev);  in team_destructor()  local
1702 	free_percpu(team->pcpu_stats);  in team_destructor()
1712 	struct team *team = netdev_priv(dev);  in team_close()  local
1715 	list_for_each_entry(port, &team->port_list, list) {  in team_close()
1728 	struct team *team = netdev_priv(dev);  in team_xmit()  local
1732 	tx_success = team_queue_override_transmit(team, skb);  in team_xmit()
1734 		tx_success = team->ops.transmit(team, skb);  in team_xmit()
1738 		pcpu_stats = this_cpu_ptr(team->pcpu_stats);  in team_xmit()
1744 		this_cpu_inc(team->pcpu_stats->tx_dropped);  in team_xmit()
1757 	 * way down to the team driver.  in team_select_queue()
1776 	struct team *team = netdev_priv(dev);  in team_change_rx_flags()  local
1781 	list_for_each_entry_rcu(port, &team->port_list, list) {  in team_change_rx_flags()
1796 	struct team *team = netdev_priv(dev);  in team_set_rx_mode()  local
1800 	list_for_each_entry_rcu(port, &team->port_list, list) {  in team_set_rx_mode()
1810 	struct team *team = netdev_priv(dev);  in team_set_mac_address()  local
1816 	mutex_lock(&team->lock);  in team_set_mac_address()
1817 	list_for_each_entry(port, &team->port_list, list)  in team_set_mac_address()
1818 		if (team->ops.port_change_dev_addr)  in team_set_mac_address()
1819 			team->ops.port_change_dev_addr(team, port);  in team_set_mac_address()
1820 	mutex_unlock(&team->lock);  in team_set_mac_address()
1826 	struct team *team = netdev_priv(dev);  in team_change_mtu()  local
1831 	 * Alhough this is reader, it's guarded by team lock. It's not possible  in team_change_mtu()
1834 	mutex_lock(&team->lock);  in team_change_mtu()
1835 	team->port_mtu_change_allowed = true;  in team_change_mtu()
1836 	list_for_each_entry(port, &team->port_list, list) {  in team_change_mtu()
1844 	team->port_mtu_change_allowed = false;  in team_change_mtu()
1845 	mutex_unlock(&team->lock);  in team_change_mtu()
1852 	list_for_each_entry_continue_reverse(port, &team->port_list, list)  in team_change_mtu()
1854 	team->port_mtu_change_allowed = false;  in team_change_mtu()
1855 	mutex_unlock(&team->lock);  in team_change_mtu()
1863 	struct team *team = netdev_priv(dev);  in team_get_stats64()  local
1871 		p = per_cpu_ptr(team->pcpu_stats, i);  in team_get_stats64()
1901 	struct team *team = netdev_priv(dev);  in team_vlan_rx_add_vid()  local
1906 	 * Alhough this is reader, it's guarded by team lock. It's not possible  in team_vlan_rx_add_vid()
1909 	mutex_lock(&team->lock);  in team_vlan_rx_add_vid()
1910 	list_for_each_entry(port, &team->port_list, list) {  in team_vlan_rx_add_vid()
1915 	mutex_unlock(&team->lock);  in team_vlan_rx_add_vid()
1920 	list_for_each_entry_continue_reverse(port, &team->port_list, list)  in team_vlan_rx_add_vid()
1922 	mutex_unlock(&team->lock);  in team_vlan_rx_add_vid()
1929 	struct team *team = netdev_priv(dev);  in team_vlan_rx_kill_vid()  local
1932 	mutex_lock(&team->lock);  in team_vlan_rx_kill_vid()
1933 	list_for_each_entry(port, &team->port_list, list)  in team_vlan_rx_kill_vid()
1935 	mutex_unlock(&team->lock);  in team_vlan_rx_kill_vid()
1945 static void __team_netpoll_cleanup(struct team *team)  in __team_netpoll_cleanup()  argument
1949 	list_for_each_entry(port, &team->port_list, list)  in __team_netpoll_cleanup()
1955 	struct team *team = netdev_priv(dev);  in team_netpoll_cleanup()  local
1957 	mutex_lock(&team->lock);  in team_netpoll_cleanup()
1958 	__team_netpoll_cleanup(team);  in team_netpoll_cleanup()
1959 	mutex_unlock(&team->lock);  in team_netpoll_cleanup()
1964 	struct team *team = netdev_priv(dev);  in team_netpoll_setup()  local
1968 	mutex_lock(&team->lock);  in team_netpoll_setup()
1969 	list_for_each_entry(port, &team->port_list, list) {  in team_netpoll_setup()
1972 			__team_netpoll_cleanup(team);  in team_netpoll_setup()
1976 	mutex_unlock(&team->lock);  in team_netpoll_setup()
1984 	struct team *team = netdev_priv(dev);  in team_add_slave()  local
1987 	mutex_lock(&team->lock);  in team_add_slave()
1988 	err = team_port_add(team, port_dev, extack);  in team_add_slave()
1989 	mutex_unlock(&team->lock);  in team_add_slave()
1999 	struct team *team = netdev_priv(dev);  in team_del_slave()  local
2002 	mutex_lock(&team->lock);  in team_del_slave()
2003 	err = team_port_del(team, port_dev);  in team_del_slave()
2004 	mutex_unlock(&team->lock);  in team_del_slave()
2010 		lockdep_unregister_key(&team->team_lock_key);  in team_del_slave()
2011 		lockdep_register_key(&team->team_lock_key);  in team_del_slave()
2012 		lockdep_set_class(&team->lock, &team->team_lock_key);  in team_del_slave()
2023 	struct team *team = netdev_priv(dev);  in team_fix_features()  local
2030 	list_for_each_entry_rcu(port, &team->port_list, list) {  in team_fix_features()
2044 	struct team *team = netdev_priv(dev);  in team_change_carrier()  local
2046 	team->user_carrier_enabled = true;  in team_change_carrier()
2094 	struct team *team= netdev_priv(dev);  in team_ethtool_get_link_ksettings()  local
2102 	list_for_each_entry_rcu(port, &team->port_list, list) {  in team_ethtool_get_link_ksettings()
2131 	struct team *team = netdev_priv(dev);  in team_setup_by_port()  local
2134 		dev->header_ops	= team->header_ops_cache;  in team_setup_by_port()
2158 	struct team *team = netdev_priv(dev);  in team_dev_type_check_change()  local
2164 	if (!list_empty(&team->port_list)) {  in team_dev_type_check_change()
2183 	struct team *team = netdev_priv(dev);  in team_setup()  local
2187 	team->header_ops_cache = dev->header_ops;  in team_setup()
2205 	/* Don't allow team devices to change network namespaces. */  in team_setup()
2255 	.priv_size		= sizeof(struct team),
2301 static struct team *team_nl_team_get(struct genl_info *info)  in team_nl_team_get()
2306 	struct team *team;  in team_nl_team_get()  local
2318 	team = netdev_priv(dev);  in team_nl_team_get()
2319 	mutex_lock(&team->lock);  in team_nl_team_get()
2320 	return team;  in team_nl_team_get()
2323 static void team_nl_team_put(struct team *team)  in team_nl_team_put()  argument
2325 	mutex_unlock(&team->lock);  in team_nl_team_put()
2326 	dev_put(team->dev);  in team_nl_team_put()
2330 				struct team *team, u32 portid);
2332 static int team_nl_send_unicast(struct sk_buff *skb, struct team *team, u32 portid)  in team_nl_send_unicast()  argument
2334 	return genlmsg_unicast(dev_net(team->dev), skb, portid);  in team_nl_send_unicast()
2337 static int team_nl_fill_one_option_get(struct sk_buff *skb, struct team *team,  in team_nl_fill_one_option_get()  argument
2347 	err = team_option_get(team, opt_inst, &ctx);  in team_nl_fill_one_option_get()
2419 				struct team *team, u32 portid,  in __send_and_alloc_skb()  argument
2425 		err = send_func(*pskb, team, portid);  in __send_and_alloc_skb()
2435 static int team_nl_send_options_get(struct team *team, u32 portid, u32 seq,  in team_nl_send_options_get()  argument
2452 	err = __send_and_alloc_skb(&skb, team, portid, send_func);  in team_nl_send_options_get()
2463 	if (nla_put_u32(skb, TEAM_ATTR_TEAM_IFINDEX, team->dev->ifindex))  in team_nl_send_options_get()
2472 		err = team_nl_fill_one_option_get(skb, team, opt_inst);  in team_nl_send_options_get()
2493 		err = __send_and_alloc_skb(&skb, team, portid, send_func);  in team_nl_send_options_get()
2499 	return send_func(skb, team, portid);  in team_nl_send_options_get()
2510 	struct team *team;  in team_nl_options_get_doit()  local
2515 	team = team_nl_team_get(info);  in team_nl_options_get_doit()
2516 	if (!team)  in team_nl_options_get_doit()
2519 	list_for_each_entry(opt_inst, &team->option_inst_list, list)  in team_nl_options_get_doit()
2521 	err = team_nl_send_options_get(team, info->snd_portid, info->snd_seq,  in team_nl_options_get_doit()
2525 	team_nl_team_put(team);  in team_nl_options_get_doit()
2530 static int team_nl_send_event_options_get(struct team *team,
2535 	struct team *team;  in team_nl_options_set_doit()  local
2542 	team = team_nl_team_get(info);  in team_nl_options_set_doit()
2543 	if (!team) {  in team_nl_options_set_doit()
2620 		list_for_each_entry(opt_inst, &team->option_inst_list, list) {  in team_nl_options_set_doit()
2663 			err = team_option_set(team, opt_inst, &ctx);  in team_nl_options_set_doit()
2674 		err = team_nl_send_event_options_get(team, &opt_inst_list);  in team_nl_options_set_doit()
2680 	team_nl_team_put(team);  in team_nl_options_set_doit()
2716 static int team_nl_send_port_list_get(struct team *team, u32 portid, u32 seq,  in team_nl_send_port_list_get()  argument
2729 	port = list_first_entry_or_null(&team->port_list,  in team_nl_send_port_list_get()
2733 	err = __send_and_alloc_skb(&skb, team, portid, send_func);  in team_nl_send_port_list_get()
2744 	if (nla_put_u32(skb, TEAM_ATTR_TEAM_IFINDEX, team->dev->ifindex))  in team_nl_send_port_list_get()
2761 		list_for_each_entry_from(port, &team->port_list, list) {  in team_nl_send_port_list_get()
2784 		err = __send_and_alloc_skb(&skb, team, portid, send_func);  in team_nl_send_port_list_get()
2790 	return send_func(skb, team, portid);  in team_nl_send_port_list_get()
2802 	struct team *team;  in team_nl_port_list_get_doit()  local
2805 	team = team_nl_team_get(info);  in team_nl_port_list_get_doit()
2806 	if (!team)  in team_nl_port_list_get_doit()
2809 	err = team_nl_send_port_list_get(team, info->snd_portid, info->snd_seq,  in team_nl_port_list_get_doit()
2812 	team_nl_team_put(team);  in team_nl_port_list_get_doit()
2836 				  struct team *team, u32 portid)  in team_nl_send_multicast()  argument
2838 	return genlmsg_multicast_netns(&team_nl_family, dev_net(team->dev),  in team_nl_send_multicast()
2842 static int team_nl_send_event_options_get(struct team *team,  in team_nl_send_event_options_get()  argument
2845 	return team_nl_send_options_get(team, 0, 0, 0, team_nl_send_multicast,  in team_nl_send_event_options_get()
2849 static int team_nl_send_event_port_get(struct team *team,  in team_nl_send_event_port_get()  argument
2852 	return team_nl_send_port_list_get(team, 0, 0, 0, team_nl_send_multicast,  in team_nl_send_event_port_get()
2871 static void __team_options_change_check(struct team *team)  in __team_options_change_check()  argument
2877 	list_for_each_entry(opt_inst, &team->option_inst_list, list) {  in __team_options_change_check()
2881 	err = team_nl_send_event_options_get(team, &sel_opt_inst_list);  in __team_options_change_check()
2883 		netdev_warn(team->dev, "Failed to send options change via netlink (err %d)\n",  in __team_options_change_check()
2910 	err = team_nl_send_event_port_get(port->team, port);  in __team_port_change_send()
2912 		netdev_warn(port->team->dev, "Failed to send port change of device %s via netlink (err %d)\n",  in __team_port_change_send()
2917 static void __team_carrier_check(struct team *team)  in __team_carrier_check()  argument
2922 	if (team->user_carrier_enabled)  in __team_carrier_check()
2926 	list_for_each_entry(port, &team->port_list, list) {  in __team_carrier_check()
2934 		netif_carrier_on(team->dev);  in __team_carrier_check()
2936 		netif_carrier_off(team->dev);  in __team_carrier_check()
2943 	__team_carrier_check(port->team);  in __team_port_change_check()
2949 	__team_carrier_check(port->team);  in __team_port_change_port_added()
2956 	__team_carrier_check(port->team);  in __team_port_change_port_removed()
2961 	struct team *team = port->team;  in team_port_change_check()  local
2963 	mutex_lock(&team->lock);  in team_port_change_check()
2965 	mutex_unlock(&team->lock);  in team_port_change_check()
2997 		team_del_slave(port->team->dev, dev);  in team_device_event()
3000 		if (!port->team->notifier_ctx) {  in team_device_event()
3001 			port->team->notifier_ctx = true;  in team_device_event()
3002 			team_compute_features(port->team);  in team_device_event()
3003 			port->team->notifier_ctx = false;  in team_device_event()
3008 		if (!port->team->port_mtu_change_allowed)  in team_device_event()
3016 		call_netdevice_notifiers(event, port->team->dev);  in team_device_event()
3068 MODULE_DESCRIPTION("Ethernet team device driver");