RDMA/uverbs: Export XRC domains to user space
[linux-3.10.git] / drivers / infiniband / core / verbs.c
1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  */
38
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/string.h>
42
43 #include <rdma/ib_verbs.h>
44 #include <rdma/ib_cache.h>
45
46 int ib_rate_to_mult(enum ib_rate rate)
47 {
48         switch (rate) {
49         case IB_RATE_2_5_GBPS: return  1;
50         case IB_RATE_5_GBPS:   return  2;
51         case IB_RATE_10_GBPS:  return  4;
52         case IB_RATE_20_GBPS:  return  8;
53         case IB_RATE_30_GBPS:  return 12;
54         case IB_RATE_40_GBPS:  return 16;
55         case IB_RATE_60_GBPS:  return 24;
56         case IB_RATE_80_GBPS:  return 32;
57         case IB_RATE_120_GBPS: return 48;
58         default:               return -1;
59         }
60 }
61 EXPORT_SYMBOL(ib_rate_to_mult);
62
63 enum ib_rate mult_to_ib_rate(int mult)
64 {
65         switch (mult) {
66         case 1:  return IB_RATE_2_5_GBPS;
67         case 2:  return IB_RATE_5_GBPS;
68         case 4:  return IB_RATE_10_GBPS;
69         case 8:  return IB_RATE_20_GBPS;
70         case 12: return IB_RATE_30_GBPS;
71         case 16: return IB_RATE_40_GBPS;
72         case 24: return IB_RATE_60_GBPS;
73         case 32: return IB_RATE_80_GBPS;
74         case 48: return IB_RATE_120_GBPS;
75         default: return IB_RATE_PORT_CURRENT;
76         }
77 }
78 EXPORT_SYMBOL(mult_to_ib_rate);
79
80 enum rdma_transport_type
81 rdma_node_get_transport(enum rdma_node_type node_type)
82 {
83         switch (node_type) {
84         case RDMA_NODE_IB_CA:
85         case RDMA_NODE_IB_SWITCH:
86         case RDMA_NODE_IB_ROUTER:
87                 return RDMA_TRANSPORT_IB;
88         case RDMA_NODE_RNIC:
89                 return RDMA_TRANSPORT_IWARP;
90         default:
91                 BUG();
92                 return 0;
93         }
94 }
95 EXPORT_SYMBOL(rdma_node_get_transport);
96
97 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
98 {
99         if (device->get_link_layer)
100                 return device->get_link_layer(device, port_num);
101
102         switch (rdma_node_get_transport(device->node_type)) {
103         case RDMA_TRANSPORT_IB:
104                 return IB_LINK_LAYER_INFINIBAND;
105         case RDMA_TRANSPORT_IWARP:
106                 return IB_LINK_LAYER_ETHERNET;
107         default:
108                 return IB_LINK_LAYER_UNSPECIFIED;
109         }
110 }
111 EXPORT_SYMBOL(rdma_port_get_link_layer);
112
113 /* Protection domains */
114
115 struct ib_pd *ib_alloc_pd(struct ib_device *device)
116 {
117         struct ib_pd *pd;
118
119         pd = device->alloc_pd(device, NULL, NULL);
120
121         if (!IS_ERR(pd)) {
122                 pd->device  = device;
123                 pd->uobject = NULL;
124                 atomic_set(&pd->usecnt, 0);
125         }
126
127         return pd;
128 }
129 EXPORT_SYMBOL(ib_alloc_pd);
130
131 int ib_dealloc_pd(struct ib_pd *pd)
132 {
133         if (atomic_read(&pd->usecnt))
134                 return -EBUSY;
135
136         return pd->device->dealloc_pd(pd);
137 }
138 EXPORT_SYMBOL(ib_dealloc_pd);
139
140 /* Address handles */
141
142 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
143 {
144         struct ib_ah *ah;
145
146         ah = pd->device->create_ah(pd, ah_attr);
147
148         if (!IS_ERR(ah)) {
149                 ah->device  = pd->device;
150                 ah->pd      = pd;
151                 ah->uobject = NULL;
152                 atomic_inc(&pd->usecnt);
153         }
154
155         return ah;
156 }
157 EXPORT_SYMBOL(ib_create_ah);
158
159 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
160                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
161 {
162         u32 flow_class;
163         u16 gid_index;
164         int ret;
165
166         memset(ah_attr, 0, sizeof *ah_attr);
167         ah_attr->dlid = wc->slid;
168         ah_attr->sl = wc->sl;
169         ah_attr->src_path_bits = wc->dlid_path_bits;
170         ah_attr->port_num = port_num;
171
172         if (wc->wc_flags & IB_WC_GRH) {
173                 ah_attr->ah_flags = IB_AH_GRH;
174                 ah_attr->grh.dgid = grh->sgid;
175
176                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
177                                          &gid_index);
178                 if (ret)
179                         return ret;
180
181                 ah_attr->grh.sgid_index = (u8) gid_index;
182                 flow_class = be32_to_cpu(grh->version_tclass_flow);
183                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
184                 ah_attr->grh.hop_limit = 0xFF;
185                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
186         }
187         return 0;
188 }
189 EXPORT_SYMBOL(ib_init_ah_from_wc);
190
191 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
192                                    struct ib_grh *grh, u8 port_num)
193 {
194         struct ib_ah_attr ah_attr;
195         int ret;
196
197         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
198         if (ret)
199                 return ERR_PTR(ret);
200
201         return ib_create_ah(pd, &ah_attr);
202 }
203 EXPORT_SYMBOL(ib_create_ah_from_wc);
204
205 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
206 {
207         return ah->device->modify_ah ?
208                 ah->device->modify_ah(ah, ah_attr) :
209                 -ENOSYS;
210 }
211 EXPORT_SYMBOL(ib_modify_ah);
212
213 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
214 {
215         return ah->device->query_ah ?
216                 ah->device->query_ah(ah, ah_attr) :
217                 -ENOSYS;
218 }
219 EXPORT_SYMBOL(ib_query_ah);
220
221 int ib_destroy_ah(struct ib_ah *ah)
222 {
223         struct ib_pd *pd;
224         int ret;
225
226         pd = ah->pd;
227         ret = ah->device->destroy_ah(ah);
228         if (!ret)
229                 atomic_dec(&pd->usecnt);
230
231         return ret;
232 }
233 EXPORT_SYMBOL(ib_destroy_ah);
234
235 /* Shared receive queues */
236
237 struct ib_srq *ib_create_srq(struct ib_pd *pd,
238                              struct ib_srq_init_attr *srq_init_attr)
239 {
240         struct ib_srq *srq;
241
242         if (!pd->device->create_srq)
243                 return ERR_PTR(-ENOSYS);
244
245         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
246
247         if (!IS_ERR(srq)) {
248                 srq->device        = pd->device;
249                 srq->pd            = pd;
250                 srq->uobject       = NULL;
251                 srq->event_handler = srq_init_attr->event_handler;
252                 srq->srq_context   = srq_init_attr->srq_context;
253                 srq->srq_type      = srq_init_attr->srq_type;
254                 if (srq->srq_type == IB_SRQT_XRC) {
255                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
256                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
257                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
258                         atomic_inc(&srq->ext.xrc.cq->usecnt);
259                 }
260                 atomic_inc(&pd->usecnt);
261                 atomic_set(&srq->usecnt, 0);
262         }
263
264         return srq;
265 }
266 EXPORT_SYMBOL(ib_create_srq);
267
268 int ib_modify_srq(struct ib_srq *srq,
269                   struct ib_srq_attr *srq_attr,
270                   enum ib_srq_attr_mask srq_attr_mask)
271 {
272         return srq->device->modify_srq ?
273                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
274                 -ENOSYS;
275 }
276 EXPORT_SYMBOL(ib_modify_srq);
277
278 int ib_query_srq(struct ib_srq *srq,
279                  struct ib_srq_attr *srq_attr)
280 {
281         return srq->device->query_srq ?
282                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
283 }
284 EXPORT_SYMBOL(ib_query_srq);
285
286 int ib_destroy_srq(struct ib_srq *srq)
287 {
288         struct ib_pd *pd;
289         enum ib_srq_type srq_type;
290         struct ib_xrcd *uninitialized_var(xrcd);
291         struct ib_cq *uninitialized_var(cq);
292         int ret;
293
294         if (atomic_read(&srq->usecnt))
295                 return -EBUSY;
296
297         pd = srq->pd;
298         srq_type = srq->srq_type;
299         if (srq_type == IB_SRQT_XRC) {
300                 xrcd = srq->ext.xrc.xrcd;
301                 cq = srq->ext.xrc.cq;
302         }
303
304         ret = srq->device->destroy_srq(srq);
305         if (!ret) {
306                 atomic_dec(&pd->usecnt);
307                 if (srq_type == IB_SRQT_XRC) {
308                         atomic_dec(&xrcd->usecnt);
309                         atomic_dec(&cq->usecnt);
310                 }
311         }
312
313         return ret;
314 }
315 EXPORT_SYMBOL(ib_destroy_srq);
316
317 /* Queue pairs */
318
319 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
320 {
321         mutex_lock(&xrcd->tgt_qp_mutex);
322         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
323         mutex_unlock(&xrcd->tgt_qp_mutex);
324 }
325
326 static void __ib_remove_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
327 {
328         mutex_lock(&xrcd->tgt_qp_mutex);
329         list_del(&qp->xrcd_list);
330         mutex_unlock(&xrcd->tgt_qp_mutex);
331 }
332
333 struct ib_qp *ib_create_qp(struct ib_pd *pd,
334                            struct ib_qp_init_attr *qp_init_attr)
335 {
336         struct ib_qp *qp;
337         struct ib_device *device;
338
339         device = pd ? pd->device : qp_init_attr->xrcd->device;
340         qp = device->create_qp(pd, qp_init_attr, NULL);
341
342         if (!IS_ERR(qp)) {
343                 qp->device = device;
344
345                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
346                         qp->pd = NULL;
347                         qp->send_cq = qp->recv_cq = NULL;
348                         qp->srq = NULL;
349                         qp->xrcd = qp_init_attr->xrcd;
350                         atomic_inc(&qp_init_attr->xrcd->usecnt);
351                         __ib_insert_xrcd_qp(qp_init_attr->xrcd, qp);
352                 } else {
353                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
354                                 qp->recv_cq = NULL;
355                                 qp->srq = NULL;
356                         } else {
357                                 qp->recv_cq = qp_init_attr->recv_cq;
358                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
359                                 qp->srq = qp_init_attr->srq;
360                                 if (qp->srq)
361                                         atomic_inc(&qp_init_attr->srq->usecnt);
362                         }
363
364                         qp->pd      = pd;
365                         qp->send_cq = qp_init_attr->send_cq;
366                         qp->xrcd    = NULL;
367
368                         atomic_inc(&pd->usecnt);
369                         atomic_inc(&qp_init_attr->send_cq->usecnt);
370                 }
371
372                 qp->uobject       = NULL;
373                 qp->event_handler = qp_init_attr->event_handler;
374                 qp->qp_context    = qp_init_attr->qp_context;
375                 qp->qp_type       = qp_init_attr->qp_type;
376         }
377
378         return qp;
379 }
380 EXPORT_SYMBOL(ib_create_qp);
381
382 static const struct {
383         int                     valid;
384         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
385         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
386 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
387         [IB_QPS_RESET] = {
388                 [IB_QPS_RESET] = { .valid = 1 },
389                 [IB_QPS_INIT]  = {
390                         .valid = 1,
391                         .req_param = {
392                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
393                                                 IB_QP_PORT                      |
394                                                 IB_QP_QKEY),
395                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
396                                                 IB_QP_PORT                      |
397                                                 IB_QP_ACCESS_FLAGS),
398                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
399                                                 IB_QP_PORT                      |
400                                                 IB_QP_ACCESS_FLAGS),
401                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
402                                                 IB_QP_PORT                      |
403                                                 IB_QP_ACCESS_FLAGS),
404                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
405                                                 IB_QP_PORT                      |
406                                                 IB_QP_ACCESS_FLAGS),
407                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
408                                                 IB_QP_QKEY),
409                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
410                                                 IB_QP_QKEY),
411                         }
412                 },
413         },
414         [IB_QPS_INIT]  = {
415                 [IB_QPS_RESET] = { .valid = 1 },
416                 [IB_QPS_ERR] =   { .valid = 1 },
417                 [IB_QPS_INIT]  = {
418                         .valid = 1,
419                         .opt_param = {
420                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
421                                                 IB_QP_PORT                      |
422                                                 IB_QP_QKEY),
423                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
424                                                 IB_QP_PORT                      |
425                                                 IB_QP_ACCESS_FLAGS),
426                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
427                                                 IB_QP_PORT                      |
428                                                 IB_QP_ACCESS_FLAGS),
429                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
430                                                 IB_QP_PORT                      |
431                                                 IB_QP_ACCESS_FLAGS),
432                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
433                                                 IB_QP_PORT                      |
434                                                 IB_QP_ACCESS_FLAGS),
435                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
436                                                 IB_QP_QKEY),
437                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
438                                                 IB_QP_QKEY),
439                         }
440                 },
441                 [IB_QPS_RTR]   = {
442                         .valid = 1,
443                         .req_param = {
444                                 [IB_QPT_UC]  = (IB_QP_AV                        |
445                                                 IB_QP_PATH_MTU                  |
446                                                 IB_QP_DEST_QPN                  |
447                                                 IB_QP_RQ_PSN),
448                                 [IB_QPT_RC]  = (IB_QP_AV                        |
449                                                 IB_QP_PATH_MTU                  |
450                                                 IB_QP_DEST_QPN                  |
451                                                 IB_QP_RQ_PSN                    |
452                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
453                                                 IB_QP_MIN_RNR_TIMER),
454                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
455                                                 IB_QP_PATH_MTU                  |
456                                                 IB_QP_DEST_QPN                  |
457                                                 IB_QP_RQ_PSN),
458                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
459                                                 IB_QP_PATH_MTU                  |
460                                                 IB_QP_DEST_QPN                  |
461                                                 IB_QP_RQ_PSN                    |
462                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
463                                                 IB_QP_MIN_RNR_TIMER),
464                         },
465                         .opt_param = {
466                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
467                                                  IB_QP_QKEY),
468                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
469                                                  IB_QP_ACCESS_FLAGS             |
470                                                  IB_QP_PKEY_INDEX),
471                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
472                                                  IB_QP_ACCESS_FLAGS             |
473                                                  IB_QP_PKEY_INDEX),
474                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
475                                                  IB_QP_ACCESS_FLAGS             |
476                                                  IB_QP_PKEY_INDEX),
477                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
478                                                  IB_QP_ACCESS_FLAGS             |
479                                                  IB_QP_PKEY_INDEX),
480                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
481                                                  IB_QP_QKEY),
482                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
483                                                  IB_QP_QKEY),
484                          }
485                 }
486         },
487         [IB_QPS_RTR]   = {
488                 [IB_QPS_RESET] = { .valid = 1 },
489                 [IB_QPS_ERR] =   { .valid = 1 },
490                 [IB_QPS_RTS]   = {
491                         .valid = 1,
492                         .req_param = {
493                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
494                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
495                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
496                                                 IB_QP_RETRY_CNT                 |
497                                                 IB_QP_RNR_RETRY                 |
498                                                 IB_QP_SQ_PSN                    |
499                                                 IB_QP_MAX_QP_RD_ATOMIC),
500                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
501                                                 IB_QP_RETRY_CNT                 |
502                                                 IB_QP_RNR_RETRY                 |
503                                                 IB_QP_SQ_PSN                    |
504                                                 IB_QP_MAX_QP_RD_ATOMIC),
505                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
506                                                 IB_QP_SQ_PSN),
507                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
508                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
509                         },
510                         .opt_param = {
511                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
512                                                  IB_QP_QKEY),
513                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
514                                                  IB_QP_ALT_PATH                 |
515                                                  IB_QP_ACCESS_FLAGS             |
516                                                  IB_QP_PATH_MIG_STATE),
517                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
518                                                  IB_QP_ALT_PATH                 |
519                                                  IB_QP_ACCESS_FLAGS             |
520                                                  IB_QP_MIN_RNR_TIMER            |
521                                                  IB_QP_PATH_MIG_STATE),
522                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
523                                                  IB_QP_ALT_PATH                 |
524                                                  IB_QP_ACCESS_FLAGS             |
525                                                  IB_QP_PATH_MIG_STATE),
526                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
527                                                  IB_QP_ALT_PATH                 |
528                                                  IB_QP_ACCESS_FLAGS             |
529                                                  IB_QP_MIN_RNR_TIMER            |
530                                                  IB_QP_PATH_MIG_STATE),
531                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
532                                                  IB_QP_QKEY),
533                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
534                                                  IB_QP_QKEY),
535                          }
536                 }
537         },
538         [IB_QPS_RTS]   = {
539                 [IB_QPS_RESET] = { .valid = 1 },
540                 [IB_QPS_ERR] =   { .valid = 1 },
541                 [IB_QPS_RTS]   = {
542                         .valid = 1,
543                         .opt_param = {
544                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
545                                                 IB_QP_QKEY),
546                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
547                                                 IB_QP_ACCESS_FLAGS              |
548                                                 IB_QP_ALT_PATH                  |
549                                                 IB_QP_PATH_MIG_STATE),
550                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
551                                                 IB_QP_ACCESS_FLAGS              |
552                                                 IB_QP_ALT_PATH                  |
553                                                 IB_QP_PATH_MIG_STATE            |
554                                                 IB_QP_MIN_RNR_TIMER),
555                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
556                                                 IB_QP_ACCESS_FLAGS              |
557                                                 IB_QP_ALT_PATH                  |
558                                                 IB_QP_PATH_MIG_STATE),
559                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
560                                                 IB_QP_ACCESS_FLAGS              |
561                                                 IB_QP_ALT_PATH                  |
562                                                 IB_QP_PATH_MIG_STATE            |
563                                                 IB_QP_MIN_RNR_TIMER),
564                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
565                                                 IB_QP_QKEY),
566                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
567                                                 IB_QP_QKEY),
568                         }
569                 },
570                 [IB_QPS_SQD]   = {
571                         .valid = 1,
572                         .opt_param = {
573                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
574                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
575                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
576                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
577                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
578                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
579                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
580                         }
581                 },
582         },
583         [IB_QPS_SQD]   = {
584                 [IB_QPS_RESET] = { .valid = 1 },
585                 [IB_QPS_ERR] =   { .valid = 1 },
586                 [IB_QPS_RTS]   = {
587                         .valid = 1,
588                         .opt_param = {
589                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
590                                                 IB_QP_QKEY),
591                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
592                                                 IB_QP_ALT_PATH                  |
593                                                 IB_QP_ACCESS_FLAGS              |
594                                                 IB_QP_PATH_MIG_STATE),
595                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
596                                                 IB_QP_ALT_PATH                  |
597                                                 IB_QP_ACCESS_FLAGS              |
598                                                 IB_QP_MIN_RNR_TIMER             |
599                                                 IB_QP_PATH_MIG_STATE),
600                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
601                                                 IB_QP_ALT_PATH                  |
602                                                 IB_QP_ACCESS_FLAGS              |
603                                                 IB_QP_PATH_MIG_STATE),
604                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
605                                                 IB_QP_ALT_PATH                  |
606                                                 IB_QP_ACCESS_FLAGS              |
607                                                 IB_QP_MIN_RNR_TIMER             |
608                                                 IB_QP_PATH_MIG_STATE),
609                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
610                                                 IB_QP_QKEY),
611                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
612                                                 IB_QP_QKEY),
613                         }
614                 },
615                 [IB_QPS_SQD]   = {
616                         .valid = 1,
617                         .opt_param = {
618                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
619                                                 IB_QP_QKEY),
620                                 [IB_QPT_UC]  = (IB_QP_AV                        |
621                                                 IB_QP_ALT_PATH                  |
622                                                 IB_QP_ACCESS_FLAGS              |
623                                                 IB_QP_PKEY_INDEX                |
624                                                 IB_QP_PATH_MIG_STATE),
625                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
626                                                 IB_QP_AV                        |
627                                                 IB_QP_TIMEOUT                   |
628                                                 IB_QP_RETRY_CNT                 |
629                                                 IB_QP_RNR_RETRY                 |
630                                                 IB_QP_MAX_QP_RD_ATOMIC          |
631                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
632                                                 IB_QP_ALT_PATH                  |
633                                                 IB_QP_ACCESS_FLAGS              |
634                                                 IB_QP_PKEY_INDEX                |
635                                                 IB_QP_MIN_RNR_TIMER             |
636                                                 IB_QP_PATH_MIG_STATE),
637                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
638                                                 IB_QP_AV                        |
639                                                 IB_QP_TIMEOUT                   |
640                                                 IB_QP_RETRY_CNT                 |
641                                                 IB_QP_RNR_RETRY                 |
642                                                 IB_QP_MAX_QP_RD_ATOMIC          |
643                                                 IB_QP_ALT_PATH                  |
644                                                 IB_QP_ACCESS_FLAGS              |
645                                                 IB_QP_PKEY_INDEX                |
646                                                 IB_QP_PATH_MIG_STATE),
647                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
648                                                 IB_QP_AV                        |
649                                                 IB_QP_TIMEOUT                   |
650                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
651                                                 IB_QP_ALT_PATH                  |
652                                                 IB_QP_ACCESS_FLAGS              |
653                                                 IB_QP_PKEY_INDEX                |
654                                                 IB_QP_MIN_RNR_TIMER             |
655                                                 IB_QP_PATH_MIG_STATE),
656                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
657                                                 IB_QP_QKEY),
658                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
659                                                 IB_QP_QKEY),
660                         }
661                 }
662         },
663         [IB_QPS_SQE]   = {
664                 [IB_QPS_RESET] = { .valid = 1 },
665                 [IB_QPS_ERR] =   { .valid = 1 },
666                 [IB_QPS_RTS]   = {
667                         .valid = 1,
668                         .opt_param = {
669                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
670                                                 IB_QP_QKEY),
671                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
672                                                 IB_QP_ACCESS_FLAGS),
673                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
674                                                 IB_QP_QKEY),
675                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
676                                                 IB_QP_QKEY),
677                         }
678                 }
679         },
680         [IB_QPS_ERR] = {
681                 [IB_QPS_RESET] = { .valid = 1 },
682                 [IB_QPS_ERR] =   { .valid = 1 }
683         }
684 };
685
686 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
687                        enum ib_qp_type type, enum ib_qp_attr_mask mask)
688 {
689         enum ib_qp_attr_mask req_param, opt_param;
690
691         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
692             next_state < 0 || next_state > IB_QPS_ERR)
693                 return 0;
694
695         if (mask & IB_QP_CUR_STATE  &&
696             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
697             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
698                 return 0;
699
700         if (!qp_state_table[cur_state][next_state].valid)
701                 return 0;
702
703         req_param = qp_state_table[cur_state][next_state].req_param[type];
704         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
705
706         if ((mask & req_param) != req_param)
707                 return 0;
708
709         if (mask & ~(req_param | opt_param | IB_QP_STATE))
710                 return 0;
711
712         return 1;
713 }
714 EXPORT_SYMBOL(ib_modify_qp_is_ok);
715
716 int ib_modify_qp(struct ib_qp *qp,
717                  struct ib_qp_attr *qp_attr,
718                  int qp_attr_mask)
719 {
720         return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL);
721 }
722 EXPORT_SYMBOL(ib_modify_qp);
723
724 int ib_query_qp(struct ib_qp *qp,
725                 struct ib_qp_attr *qp_attr,
726                 int qp_attr_mask,
727                 struct ib_qp_init_attr *qp_init_attr)
728 {
729         return qp->device->query_qp ?
730                 qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) :
731                 -ENOSYS;
732 }
733 EXPORT_SYMBOL(ib_query_qp);
734
735 int ib_destroy_qp(struct ib_qp *qp)
736 {
737         struct ib_pd *pd;
738         struct ib_cq *scq, *rcq;
739         struct ib_srq *srq;
740         struct ib_xrcd *xrcd;
741         int ret;
742
743         pd   = qp->pd;
744         scq  = qp->send_cq;
745         rcq  = qp->recv_cq;
746         srq  = qp->srq;
747         xrcd = qp->xrcd;
748         if (xrcd)
749                 __ib_remove_xrcd_qp(xrcd, qp);
750
751         ret = qp->device->destroy_qp(qp);
752         if (!ret) {
753                 if (pd)
754                         atomic_dec(&pd->usecnt);
755                 if (scq)
756                         atomic_dec(&scq->usecnt);
757                 if (rcq)
758                         atomic_dec(&rcq->usecnt);
759                 if (srq)
760                         atomic_dec(&srq->usecnt);
761                 if (xrcd)
762                         atomic_dec(&xrcd->usecnt);
763         } else if (xrcd) {
764                 __ib_insert_xrcd_qp(xrcd, qp);
765         }
766
767         return ret;
768 }
769 EXPORT_SYMBOL(ib_destroy_qp);
770
771 int ib_release_qp(struct ib_qp *qp)
772 {
773         unsigned long flags;
774
775         if (qp->qp_type != IB_QPT_XRC_TGT)
776                 return -EINVAL;
777
778         spin_lock_irqsave(&qp->device->event_handler_lock, flags);
779         qp->event_handler = NULL;
780         spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
781
782         atomic_dec(&qp->xrcd->usecnt);
783         return 0;
784 }
785 EXPORT_SYMBOL(ib_release_qp);
786
787 /* Completion queues */
788
789 struct ib_cq *ib_create_cq(struct ib_device *device,
790                            ib_comp_handler comp_handler,
791                            void (*event_handler)(struct ib_event *, void *),
792                            void *cq_context, int cqe, int comp_vector)
793 {
794         struct ib_cq *cq;
795
796         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
797
798         if (!IS_ERR(cq)) {
799                 cq->device        = device;
800                 cq->uobject       = NULL;
801                 cq->comp_handler  = comp_handler;
802                 cq->event_handler = event_handler;
803                 cq->cq_context    = cq_context;
804                 atomic_set(&cq->usecnt, 0);
805         }
806
807         return cq;
808 }
809 EXPORT_SYMBOL(ib_create_cq);
810
811 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
812 {
813         return cq->device->modify_cq ?
814                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
815 }
816 EXPORT_SYMBOL(ib_modify_cq);
817
818 int ib_destroy_cq(struct ib_cq *cq)
819 {
820         if (atomic_read(&cq->usecnt))
821                 return -EBUSY;
822
823         return cq->device->destroy_cq(cq);
824 }
825 EXPORT_SYMBOL(ib_destroy_cq);
826
827 int ib_resize_cq(struct ib_cq *cq, int cqe)
828 {
829         return cq->device->resize_cq ?
830                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
831 }
832 EXPORT_SYMBOL(ib_resize_cq);
833
834 /* Memory regions */
835
836 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
837 {
838         struct ib_mr *mr;
839
840         mr = pd->device->get_dma_mr(pd, mr_access_flags);
841
842         if (!IS_ERR(mr)) {
843                 mr->device  = pd->device;
844                 mr->pd      = pd;
845                 mr->uobject = NULL;
846                 atomic_inc(&pd->usecnt);
847                 atomic_set(&mr->usecnt, 0);
848         }
849
850         return mr;
851 }
852 EXPORT_SYMBOL(ib_get_dma_mr);
853
854 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
855                              struct ib_phys_buf *phys_buf_array,
856                              int num_phys_buf,
857                              int mr_access_flags,
858                              u64 *iova_start)
859 {
860         struct ib_mr *mr;
861
862         if (!pd->device->reg_phys_mr)
863                 return ERR_PTR(-ENOSYS);
864
865         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
866                                      mr_access_flags, iova_start);
867
868         if (!IS_ERR(mr)) {
869                 mr->device  = pd->device;
870                 mr->pd      = pd;
871                 mr->uobject = NULL;
872                 atomic_inc(&pd->usecnt);
873                 atomic_set(&mr->usecnt, 0);
874         }
875
876         return mr;
877 }
878 EXPORT_SYMBOL(ib_reg_phys_mr);
879
880 int ib_rereg_phys_mr(struct ib_mr *mr,
881                      int mr_rereg_mask,
882                      struct ib_pd *pd,
883                      struct ib_phys_buf *phys_buf_array,
884                      int num_phys_buf,
885                      int mr_access_flags,
886                      u64 *iova_start)
887 {
888         struct ib_pd *old_pd;
889         int ret;
890
891         if (!mr->device->rereg_phys_mr)
892                 return -ENOSYS;
893
894         if (atomic_read(&mr->usecnt))
895                 return -EBUSY;
896
897         old_pd = mr->pd;
898
899         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
900                                         phys_buf_array, num_phys_buf,
901                                         mr_access_flags, iova_start);
902
903         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
904                 atomic_dec(&old_pd->usecnt);
905                 atomic_inc(&pd->usecnt);
906         }
907
908         return ret;
909 }
910 EXPORT_SYMBOL(ib_rereg_phys_mr);
911
912 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
913 {
914         return mr->device->query_mr ?
915                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
916 }
917 EXPORT_SYMBOL(ib_query_mr);
918
919 int ib_dereg_mr(struct ib_mr *mr)
920 {
921         struct ib_pd *pd;
922         int ret;
923
924         if (atomic_read(&mr->usecnt))
925                 return -EBUSY;
926
927         pd = mr->pd;
928         ret = mr->device->dereg_mr(mr);
929         if (!ret)
930                 atomic_dec(&pd->usecnt);
931
932         return ret;
933 }
934 EXPORT_SYMBOL(ib_dereg_mr);
935
936 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
937 {
938         struct ib_mr *mr;
939
940         if (!pd->device->alloc_fast_reg_mr)
941                 return ERR_PTR(-ENOSYS);
942
943         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
944
945         if (!IS_ERR(mr)) {
946                 mr->device  = pd->device;
947                 mr->pd      = pd;
948                 mr->uobject = NULL;
949                 atomic_inc(&pd->usecnt);
950                 atomic_set(&mr->usecnt, 0);
951         }
952
953         return mr;
954 }
955 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
956
957 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
958                                                           int max_page_list_len)
959 {
960         struct ib_fast_reg_page_list *page_list;
961
962         if (!device->alloc_fast_reg_page_list)
963                 return ERR_PTR(-ENOSYS);
964
965         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
966
967         if (!IS_ERR(page_list)) {
968                 page_list->device = device;
969                 page_list->max_page_list_len = max_page_list_len;
970         }
971
972         return page_list;
973 }
974 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
975
976 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
977 {
978         page_list->device->free_fast_reg_page_list(page_list);
979 }
980 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
981
982 /* Memory windows */
983
984 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
985 {
986         struct ib_mw *mw;
987
988         if (!pd->device->alloc_mw)
989                 return ERR_PTR(-ENOSYS);
990
991         mw = pd->device->alloc_mw(pd);
992         if (!IS_ERR(mw)) {
993                 mw->device  = pd->device;
994                 mw->pd      = pd;
995                 mw->uobject = NULL;
996                 atomic_inc(&pd->usecnt);
997         }
998
999         return mw;
1000 }
1001 EXPORT_SYMBOL(ib_alloc_mw);
1002
1003 int ib_dealloc_mw(struct ib_mw *mw)
1004 {
1005         struct ib_pd *pd;
1006         int ret;
1007
1008         pd = mw->pd;
1009         ret = mw->device->dealloc_mw(mw);
1010         if (!ret)
1011                 atomic_dec(&pd->usecnt);
1012
1013         return ret;
1014 }
1015 EXPORT_SYMBOL(ib_dealloc_mw);
1016
1017 /* "Fast" memory regions */
1018
1019 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1020                             int mr_access_flags,
1021                             struct ib_fmr_attr *fmr_attr)
1022 {
1023         struct ib_fmr *fmr;
1024
1025         if (!pd->device->alloc_fmr)
1026                 return ERR_PTR(-ENOSYS);
1027
1028         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1029         if (!IS_ERR(fmr)) {
1030                 fmr->device = pd->device;
1031                 fmr->pd     = pd;
1032                 atomic_inc(&pd->usecnt);
1033         }
1034
1035         return fmr;
1036 }
1037 EXPORT_SYMBOL(ib_alloc_fmr);
1038
1039 int ib_unmap_fmr(struct list_head *fmr_list)
1040 {
1041         struct ib_fmr *fmr;
1042
1043         if (list_empty(fmr_list))
1044                 return 0;
1045
1046         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1047         return fmr->device->unmap_fmr(fmr_list);
1048 }
1049 EXPORT_SYMBOL(ib_unmap_fmr);
1050
1051 int ib_dealloc_fmr(struct ib_fmr *fmr)
1052 {
1053         struct ib_pd *pd;
1054         int ret;
1055
1056         pd = fmr->pd;
1057         ret = fmr->device->dealloc_fmr(fmr);
1058         if (!ret)
1059                 atomic_dec(&pd->usecnt);
1060
1061         return ret;
1062 }
1063 EXPORT_SYMBOL(ib_dealloc_fmr);
1064
1065 /* Multicast groups */
1066
1067 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1068 {
1069         if (!qp->device->attach_mcast)
1070                 return -ENOSYS;
1071         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1072                 return -EINVAL;
1073
1074         return qp->device->attach_mcast(qp, gid, lid);
1075 }
1076 EXPORT_SYMBOL(ib_attach_mcast);
1077
1078 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1079 {
1080         if (!qp->device->detach_mcast)
1081                 return -ENOSYS;
1082         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1083                 return -EINVAL;
1084
1085         return qp->device->detach_mcast(qp, gid, lid);
1086 }
1087 EXPORT_SYMBOL(ib_detach_mcast);
1088
1089 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1090 {
1091         struct ib_xrcd *xrcd;
1092
1093         if (!device->alloc_xrcd)
1094                 return ERR_PTR(-ENOSYS);
1095
1096         xrcd = device->alloc_xrcd(device, NULL, NULL);
1097         if (!IS_ERR(xrcd)) {
1098                 xrcd->device = device;
1099                 xrcd->inode = NULL;
1100                 atomic_set(&xrcd->usecnt, 0);
1101                 mutex_init(&xrcd->tgt_qp_mutex);
1102                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1103         }
1104
1105         return xrcd;
1106 }
1107 EXPORT_SYMBOL(ib_alloc_xrcd);
1108
1109 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1110 {
1111         struct ib_qp *qp;
1112         int ret;
1113
1114         if (atomic_read(&xrcd->usecnt))
1115                 return -EBUSY;
1116
1117         while (!list_empty(&xrcd->tgt_qp_list)) {
1118                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1119                 ret = ib_destroy_qp(qp);
1120                 if (ret)
1121                         return ret;
1122         }
1123
1124         return xrcd->device->dealloc_xrcd(xrcd);
1125 }
1126 EXPORT_SYMBOL(ib_dealloc_xrcd);