1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
22 #include <rdma/mlx5-abi.h>
32 	dev_dbg(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,      \
33 		__LINE__, current->pid, ##arg)
36 	dev_err(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,      \
37 		__LINE__, current->pid, ##arg)
40 	dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,     \
41 		 __LINE__, current->pid, ##arg)
44 	dev_printk(lvl, &(_dev)->ib_dev.dev,  "%s:%d:(pid %d): " format,       \
45 		   __func__, __LINE__, current->pid, ##arg)
55 		min_t(unsigned long, (1ULL << log_pgsz_bits) - 1 + pgsz_shift,  in __mlx5_log_page_size_to_bitmap()
56 		      BITS_PER_LONG - 1);  in __mlx5_log_page_size_to_bitmap()
71 		min_t(unsigned long, page_offset_bits - 1 + offset_shift,  in __mlx5_page_offset_to_bitmask()
72 		      BITS_PER_LONG - 1);  in __mlx5_page_offset_to_bitmask()
116 	return ib_umem_find_best_pgsz(&umem_dmabuf->umem, PAGE_SIZE,  in mlx5_umem_dmabuf_find_best_pgsz()
117 				      umem_dmabuf->umem.iova);  in mlx5_umem_dmabuf_find_best_pgsz()
233 #define MLX5_IB_FLOW_MCAST_PRIO		(MLX5_BY_PASS_NUM_PRIOS - 1)
234 #define MLX5_IB_FLOW_LAST_PRIO		(MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
319  * enum ib_qp_type for low-level driver
538 	 * IB/core doesn't store low-level QP types, so
638 	u8 ats:1;  member
670 	atomic64_add(value, &((mr)->odp_stats.counter_name))
675 		atomic64_add(1, &((mr)->odp_stats.counter_name##_handled)); \
733 	return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&  in is_odp_mr()
734 	       mr->umem->is_odp;  in is_odp_mr()
739 	return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&  in is_dmabuf_mr()
740 	       mr->umem->is_dmabuf;  in is_dmabuf_mr()
777 	((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))
807 	 * - limit is the low water mark for stored mkeys, 2* limit is the
1062 	/* number of counters supported for this counters type */
1229 	return to_mdev(mr->ibmr.device);  in mr_to_mdev()
1237 	return to_mdev(context->ibucontext.device);  in mlx5_udata_to_mdev()
1247 	return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;  in to_mibqp()
1483 	return -EOPNOTSUPP;  in mlx5_ib_advise_mr_prefetch()
1487 	return -EOPNOTSUPP;  in mlx5_ib_init_odp_mr()
1491 	return -EOPNOTSUPP;  in mlx5_ib_init_dmabuf_mr()
1562 	 * It returns non-zero value for unsupported CQ  in check_cq_create_flags()
1575 			return -EINVAL;  in verify_assign_uidx()
1589 	u8 cqe_version = ucontext->cqe_version;  in get_qp_user_index()
1592 	    (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))  in get_qp_user_index()
1596 		return -EINVAL;  in get_qp_user_index()
1598 	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);  in get_qp_user_index()
1606 	u8 cqe_version = ucontext->cqe_version;  in get_srq_user_index()
1609 	    (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))  in get_srq_user_index()
1613 		return -EINVAL;  in get_srq_user_index()
1615 	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);  in get_srq_user_index()
1620 	return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?  in get_uars_per_sys_page()
1633 	refcount_set(&mmkey->usecount, 1);  in mlx5r_store_odp_mkey()
1635 	return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mmkey->key),  in mlx5r_store_odp_mkey()
1642 	if (refcount_dec_and_test(&mmkey->usecount))  in mlx5r_deref_odp_mkey()
1643 		wake_up(&mmkey->wait);  in mlx5r_deref_odp_mkey()
1650 	wait_event(mmkey->wait, refcount_read(&mmkey->usecount) == 0);  in mlx5r_deref_wait_odp_mkey()
1657 	 * is supported, it means that the driver no longer needs to assign the port  in mlx5_ib_lag_should_assign_affinity()
1662 	if (dev->lag_active &&  in mlx5_ib_lag_should_assign_affinity()
1663 	    mlx5_lag_mode_is_hash(dev->mdev) &&  in mlx5_ib_lag_should_assign_affinity()
1664 	    MLX5_CAP_PORT_SELECTION(dev->mdev, port_select_flow_table_bypass))  in mlx5_ib_lag_should_assign_affinity()
1667 	if (mlx5_lag_is_lacp_owner(dev->mdev) && !dev->lag_active)  in mlx5_ib_lag_should_assign_affinity()
1670 	return dev->lag_active ||  in mlx5_ib_lag_should_assign_affinity()
1671 		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&  in mlx5_ib_lag_should_assign_affinity()
1672 		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));  in mlx5_ib_lag_should_assign_affinity()
1687  * perfectly - so any scenario where it sees only half the transaction is a
1690  * CR/RR/DT  ATS RO P2P
1704  * CR/RR/DT is 111 if the ATS cap is enabled and follow the last three rows.
1711 	if (!MLX5_CAP_GEN(dev->mdev, ats) || !umem->is_dmabuf)  in mlx5_umem_needs_ats()
1722 	return (port - 1) / dev->num_ports + 1;  in smi_to_native_portnum()
1727  * specifies both the page_offset and the on-the-wire IOVA
1734 		MLX5_CAP_GEN_2(dev->mdev, umr_log_entity_size_5) ? 6 : 5;  in mlx5_umem_mkc_find_best_pgsz()