1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2
3 /* Authors: Cheng Xu <[email protected]> */
4 /* Kai Shen <[email protected]> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 #ifndef __ERDMA_VERBS_H__
8 #define __ERDMA_VERBS_H__
9
10 #include "erdma.h"
11
12 /* RDMA Capability. */
13 #define ERDMA_MAX_PD (128 * 1024)
14 #define ERDMA_MAX_SEND_WR 8192
15 #define ERDMA_MAX_ORD 128
16 #define ERDMA_MAX_IRD 128
17 #define ERDMA_MAX_SGE_RD 1
18 #define ERDMA_MAX_CONTEXT (128 * 1024)
19 #define ERDMA_MAX_SEND_SGE 6
20 #define ERDMA_MAX_RECV_SGE 1
21 #define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
22 #define ERDMA_MAX_FRMR_PA 512
23
24 enum {
25 ERDMA_MMAP_IO_NC = 0, /* no cache */
26 };
27
28 struct erdma_user_mmap_entry {
29 struct rdma_user_mmap_entry rdma_entry;
30 u64 address;
31 u8 mmap_flag;
32 };
33
34 struct erdma_ext_db_info {
35 bool enable;
36 u16 sdb_off;
37 u16 rdb_off;
38 u16 cdb_off;
39 };
40
41 struct erdma_ucontext {
42 struct ib_ucontext ibucontext;
43
44 struct erdma_ext_db_info ext_db;
45
46 u64 sdb;
47 u64 rdb;
48 u64 cdb;
49
50 struct rdma_user_mmap_entry *sq_db_mmap_entry;
51 struct rdma_user_mmap_entry *rq_db_mmap_entry;
52 struct rdma_user_mmap_entry *cq_db_mmap_entry;
53
54 /* doorbell records */
55 struct list_head dbrecords_page_list;
56 struct mutex dbrecords_page_mutex;
57 };
58
59 struct erdma_pd {
60 struct ib_pd ibpd;
61 u32 pdn;
62 };
63
64 /*
65 * MemoryRegion definition.
66 */
67 #define ERDMA_MAX_INLINE_MTT_ENTRIES 4
68 #define MTT_SIZE(mtt_cnt) ((mtt_cnt) << 3) /* per mtt entry takes 8 Bytes. */
69 #define ERDMA_MR_MAX_MTT_CNT 524288
70 #define ERDMA_MTT_ENTRY_SIZE 8
71
72 #define ERDMA_MR_TYPE_NORMAL 0
73 #define ERDMA_MR_TYPE_FRMR 1
74 #define ERDMA_MR_TYPE_DMA 2
75
76 #define ERDMA_MR_MTT_0LEVEL 0
77 #define ERDMA_MR_MTT_1LEVEL 1
78
79 #define ERDMA_MR_ACC_RA BIT(0)
80 #define ERDMA_MR_ACC_LR BIT(1)
81 #define ERDMA_MR_ACC_LW BIT(2)
82 #define ERDMA_MR_ACC_RR BIT(3)
83 #define ERDMA_MR_ACC_RW BIT(4)
84
to_erdma_access_flags(int access)85 static inline u8 to_erdma_access_flags(int access)
86 {
87 return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
88 (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
89 (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) |
90 (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);
91 }
92
93 /* Hierarchical storage structure for MTT entries */
94 struct erdma_mtt {
95 u64 *buf;
96 size_t size;
97
98 bool continuous;
99 union {
100 dma_addr_t buf_dma;
101 struct {
102 struct scatterlist *sglist;
103 u32 nsg;
104 u32 level;
105 };
106 };
107
108 struct erdma_mtt *low_level;
109 };
110
111 struct erdma_mem {
112 struct ib_umem *umem;
113 struct erdma_mtt *mtt;
114
115 u32 page_size;
116 u32 page_offset;
117 u32 page_cnt;
118 u32 mtt_nents;
119
120 u64 va;
121 u64 len;
122 };
123
124 struct erdma_mr {
125 struct ib_mr ibmr;
126 struct erdma_mem mem;
127 u8 type;
128 u8 access;
129 u8 valid;
130 };
131
132 struct erdma_user_dbrecords_page {
133 struct list_head list;
134 struct ib_umem *umem;
135 u64 va;
136 int refcnt;
137 };
138
139 struct erdma_av {
140 u8 port;
141 u8 hop_limit;
142 u8 traffic_class;
143 u8 sl;
144 u8 sgid_index;
145 u16 udp_sport;
146 u32 flow_label;
147 u8 dmac[ETH_ALEN];
148 u8 dgid[ERDMA_ROCEV2_GID_SIZE];
149 enum erdma_network_type ntype;
150 };
151
152 struct erdma_ah {
153 struct ib_ah ibah;
154 struct erdma_av av;
155 u32 ahn;
156 };
157
158 struct erdma_uqp {
159 struct erdma_mem sq_mem;
160 struct erdma_mem rq_mem;
161
162 dma_addr_t sq_dbrec_dma;
163 dma_addr_t rq_dbrec_dma;
164
165 struct erdma_user_dbrecords_page *user_dbr_page;
166
167 u32 rq_offset;
168 };
169
170 struct erdma_kqp {
171 u16 sq_pi;
172 u16 sq_ci;
173
174 u16 rq_pi;
175 u16 rq_ci;
176
177 u64 *swr_tbl;
178 u64 *rwr_tbl;
179
180 void __iomem *hw_sq_db;
181 void __iomem *hw_rq_db;
182
183 void *sq_buf;
184 dma_addr_t sq_buf_dma_addr;
185
186 void *rq_buf;
187 dma_addr_t rq_buf_dma_addr;
188
189 void *sq_dbrec;
190 void *rq_dbrec;
191
192 dma_addr_t sq_dbrec_dma;
193 dma_addr_t rq_dbrec_dma;
194
195 u8 sig_all;
196 };
197
198 enum erdma_qps_iwarp {
199 ERDMA_QPS_IWARP_IDLE = 0,
200 ERDMA_QPS_IWARP_RTR = 1,
201 ERDMA_QPS_IWARP_RTS = 2,
202 ERDMA_QPS_IWARP_CLOSING = 3,
203 ERDMA_QPS_IWARP_TERMINATE = 4,
204 ERDMA_QPS_IWARP_ERROR = 5,
205 ERDMA_QPS_IWARP_UNDEF = 6,
206 ERDMA_QPS_IWARP_COUNT = 7,
207 };
208
209 enum erdma_qpa_mask_iwarp {
210 ERDMA_QPA_IWARP_STATE = (1 << 0),
211 ERDMA_QPA_IWARP_LLP_HANDLE = (1 << 2),
212 ERDMA_QPA_IWARP_ORD = (1 << 3),
213 ERDMA_QPA_IWARP_IRD = (1 << 4),
214 ERDMA_QPA_IWARP_SQ_SIZE = (1 << 5),
215 ERDMA_QPA_IWARP_RQ_SIZE = (1 << 6),
216 ERDMA_QPA_IWARP_MPA = (1 << 7),
217 ERDMA_QPA_IWARP_CC = (1 << 8),
218 };
219
220 enum erdma_qps_rocev2 {
221 ERDMA_QPS_ROCEV2_RESET = 0,
222 ERDMA_QPS_ROCEV2_INIT = 1,
223 ERDMA_QPS_ROCEV2_RTR = 2,
224 ERDMA_QPS_ROCEV2_RTS = 3,
225 ERDMA_QPS_ROCEV2_SQD = 4,
226 ERDMA_QPS_ROCEV2_SQE = 5,
227 ERDMA_QPS_ROCEV2_ERROR = 6,
228 ERDMA_QPS_ROCEV2_COUNT = 7,
229 };
230
231 enum erdma_qpa_mask_rocev2 {
232 ERDMA_QPA_ROCEV2_STATE = (1 << 0),
233 ERDMA_QPA_ROCEV2_QKEY = (1 << 1),
234 ERDMA_QPA_ROCEV2_AV = (1 << 2),
235 ERDMA_QPA_ROCEV2_SQ_PSN = (1 << 3),
236 ERDMA_QPA_ROCEV2_RQ_PSN = (1 << 4),
237 ERDMA_QPA_ROCEV2_DST_QPN = (1 << 5),
238 };
239
240 enum erdma_qp_flags {
241 ERDMA_QP_IN_FLUSHING = (1 << 0),
242 };
243
244 #define ERDMA_QP_ACTIVE 0
245 #define ERDMA_QP_PASSIVE 1
246
247 struct erdma_mod_qp_params_iwarp {
248 enum erdma_qps_iwarp state;
249 enum erdma_cc_alg cc;
250 u8 qp_type;
251 u8 pd_len;
252 u32 irq_size;
253 u32 orq_size;
254 };
255
256 struct erdma_qp_attrs_iwarp {
257 enum erdma_qps_iwarp state;
258 u32 cookie;
259 };
260
261 struct erdma_mod_qp_params_rocev2 {
262 enum erdma_qps_rocev2 state;
263 u32 qkey;
264 u32 sq_psn;
265 u32 rq_psn;
266 u32 dst_qpn;
267 struct erdma_av av;
268 };
269
270 union erdma_mod_qp_params {
271 struct erdma_mod_qp_params_iwarp iwarp;
272 struct erdma_mod_qp_params_rocev2 rocev2;
273 };
274
275 struct erdma_qp_attrs_rocev2 {
276 enum erdma_qps_rocev2 state;
277 u32 qkey;
278 u32 dst_qpn;
279 struct erdma_av av;
280 };
281
282 struct erdma_qp_attrs {
283 enum erdma_cc_alg cc; /* Congestion control algorithm */
284 u32 sq_size;
285 u32 rq_size;
286 u32 orq_size;
287 u32 irq_size;
288 u32 max_send_sge;
289 u32 max_recv_sge;
290 union {
291 struct erdma_qp_attrs_iwarp iwarp;
292 struct erdma_qp_attrs_rocev2 rocev2;
293 };
294 };
295
296 struct erdma_qp {
297 struct ib_qp ibqp;
298 struct kref ref;
299 struct completion safe_free;
300 struct erdma_dev *dev;
301 struct erdma_cep *cep;
302 struct rw_semaphore state_lock;
303
304 unsigned long flags;
305 struct delayed_work reflush_dwork;
306
307 union {
308 struct erdma_kqp kern_qp;
309 struct erdma_uqp user_qp;
310 };
311
312 struct erdma_cq *scq;
313 struct erdma_cq *rcq;
314
315 struct erdma_qp_attrs attrs;
316 spinlock_t lock;
317 };
318
319 struct erdma_kcq_info {
320 void *qbuf;
321 dma_addr_t qbuf_dma_addr;
322 u32 ci;
323 u32 cmdsn;
324 u32 notify_cnt;
325
326 spinlock_t lock;
327 u8 __iomem *db;
328 u64 *dbrec;
329 dma_addr_t dbrec_dma;
330 };
331
332 struct erdma_ucq_info {
333 struct erdma_mem qbuf_mem;
334 struct erdma_user_dbrecords_page *user_dbr_page;
335 dma_addr_t dbrec_dma;
336 };
337
338 struct erdma_cq {
339 struct ib_cq ibcq;
340 u32 cqn;
341
342 u32 depth;
343 u32 assoc_eqn;
344
345 union {
346 struct erdma_kcq_info kern_cq;
347 struct erdma_ucq_info user_cq;
348 };
349 };
350
351 #define QP_ID(qp) ((qp)->ibqp.qp_num)
352
find_qp_by_qpn(struct erdma_dev * dev,int id)353 static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
354 {
355 return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
356 }
357
find_cq_by_cqn(struct erdma_dev * dev,int id)358 static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
359 {
360 return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
361 }
362
363 void erdma_qp_get(struct erdma_qp *qp);
364 void erdma_qp_put(struct erdma_qp *qp);
365 int erdma_modify_qp_state_iwarp(struct erdma_qp *qp,
366 struct erdma_mod_qp_params_iwarp *params,
367 int mask);
368 int erdma_modify_qp_state_rocev2(struct erdma_qp *qp,
369 struct erdma_mod_qp_params_rocev2 *params,
370 int attr_mask);
371 void erdma_qp_llp_close(struct erdma_qp *qp);
372 void erdma_qp_cm_drop(struct erdma_qp *qp);
373
erdma_device_iwarp(struct erdma_dev * dev)374 static inline bool erdma_device_iwarp(struct erdma_dev *dev)
375 {
376 return dev->proto == ERDMA_PROTO_IWARP;
377 }
378
erdma_device_rocev2(struct erdma_dev * dev)379 static inline bool erdma_device_rocev2(struct erdma_dev *dev)
380 {
381 return dev->proto == ERDMA_PROTO_ROCEV2;
382 }
383
to_ectx(struct ib_ucontext * ibctx)384 static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
385 {
386 return container_of(ibctx, struct erdma_ucontext, ibucontext);
387 }
388
to_epd(struct ib_pd * pd)389 static inline struct erdma_pd *to_epd(struct ib_pd *pd)
390 {
391 return container_of(pd, struct erdma_pd, ibpd);
392 }
393
to_emr(struct ib_mr * ibmr)394 static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
395 {
396 return container_of(ibmr, struct erdma_mr, ibmr);
397 }
398
to_eqp(struct ib_qp * qp)399 static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
400 {
401 return container_of(qp, struct erdma_qp, ibqp);
402 }
403
to_ecq(struct ib_cq * ibcq)404 static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
405 {
406 return container_of(ibcq, struct erdma_cq, ibcq);
407 }
408
to_eah(struct ib_ah * ibah)409 static inline struct erdma_ah *to_eah(struct ib_ah *ibah)
410 {
411 return container_of(ibah, struct erdma_ah, ibah);
412 }
413
erdma_check_gid_attr(const struct ib_gid_attr * attr)414 static inline int erdma_check_gid_attr(const struct ib_gid_attr *attr)
415 {
416 u8 ntype = rdma_gid_attr_network_type(attr);
417
418 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6)
419 return -EINVAL;
420
421 return 0;
422 }
423
424 static inline struct erdma_user_mmap_entry *
to_emmap(struct rdma_user_mmap_entry * ibmmap)425 to_emmap(struct rdma_user_mmap_entry *ibmmap)
426 {
427 return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
428 }
429
430 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);
431 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);
432 int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
433 struct ib_udata *data);
434 int erdma_get_port_immutable(struct ib_device *dev, u32 port,
435 struct ib_port_immutable *ib_port_immutable);
436 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
437 struct uverbs_attr_bundle *attrs);
438 int erdma_query_port(struct ib_device *dev, u32 port,
439 struct ib_port_attr *attr);
440 int erdma_query_gid(struct ib_device *dev, u32 port, int idx,
441 union ib_gid *gid);
442 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);
443 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
444 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
445 struct ib_udata *data);
446 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
447 struct ib_qp_init_attr *init_attr);
448 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
449 struct ib_udata *data);
450 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
451 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
452 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext);
453 int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
454 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
455 u64 virt, int access, struct ib_udata *udata);
456 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
457 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
458 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
459 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
460 void erdma_qp_get_ref(struct ib_qp *ibqp);
461 void erdma_qp_put_ref(struct ib_qp *ibqp);
462 struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
463 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
464 const struct ib_send_wr **bad_send_wr);
465 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
466 const struct ib_recv_wr **bad_recv_wr);
467 int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
468 void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn);
469 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
470 u32 max_num_sg);
471 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
472 unsigned int *sg_offset);
473 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
474 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu);
475 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
476 u32 port_num);
477 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
478 u32 port, int index);
479 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev,
480 u32 port_num);
481 int erdma_add_gid(const struct ib_gid_attr *attr, void **context);
482 int erdma_del_gid(const struct ib_gid_attr *attr, void **context);
483 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
484 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av);
485 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
486 struct ib_udata *udata);
487 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags);
488 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
489
490 #endif
491