RDMA/addr: Keep pointer to netdevice in struct rdma_dev_addr
[linux-3.10.git] / drivers / infiniband / core / cma.c
1 /*
2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include <linux/completion.h>
37 #include <linux/in.h>
38 #include <linux/in6.h>
39 #include <linux/mutex.h>
40 #include <linux/random.h>
41 #include <linux/idr.h>
42 #include <linux/inetdevice.h>
43
44 #include <net/tcp.h>
45
46 #include <rdma/rdma_cm.h>
47 #include <rdma/rdma_cm_ib.h>
48 #include <rdma/ib_cache.h>
49 #include <rdma/ib_cm.h>
50 #include <rdma/ib_sa.h>
51 #include <rdma/iw_cm.h>
52
53 MODULE_AUTHOR("Sean Hefty");
54 MODULE_DESCRIPTION("Generic RDMA CM Agent");
55 MODULE_LICENSE("Dual BSD/GPL");
56
57 #define CMA_CM_RESPONSE_TIMEOUT 20
58 #define CMA_MAX_CM_RETRIES 15
59 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
60
61 static void cma_add_one(struct ib_device *device);
62 static void cma_remove_one(struct ib_device *device);
63
64 static struct ib_client cma_client = {
65         .name   = "cma",
66         .add    = cma_add_one,
67         .remove = cma_remove_one
68 };
69
70 static struct ib_sa_client sa_client;
71 static struct rdma_addr_client addr_client;
72 static LIST_HEAD(dev_list);
73 static LIST_HEAD(listen_any_list);
74 static DEFINE_MUTEX(lock);
75 static struct workqueue_struct *cma_wq;
76 static DEFINE_IDR(sdp_ps);
77 static DEFINE_IDR(tcp_ps);
78 static DEFINE_IDR(udp_ps);
79 static DEFINE_IDR(ipoib_ps);
80 static int next_port;
81
82 struct cma_device {
83         struct list_head        list;
84         struct ib_device        *device;
85         struct completion       comp;
86         atomic_t                refcount;
87         struct list_head        id_list;
88 };
89
90 enum cma_state {
91         CMA_IDLE,
92         CMA_ADDR_QUERY,
93         CMA_ADDR_RESOLVED,
94         CMA_ROUTE_QUERY,
95         CMA_ROUTE_RESOLVED,
96         CMA_CONNECT,
97         CMA_DISCONNECT,
98         CMA_ADDR_BOUND,
99         CMA_LISTEN,
100         CMA_DEVICE_REMOVAL,
101         CMA_DESTROYING
102 };
103
104 struct rdma_bind_list {
105         struct idr              *ps;
106         struct hlist_head       owners;
107         unsigned short          port;
108 };
109
110 /*
111  * Device removal can occur at anytime, so we need extra handling to
112  * serialize notifying the user of device removal with other callbacks.
113  * We do this by disabling removal notification while a callback is in process,
114  * and reporting it after the callback completes.
115  */
116 struct rdma_id_private {
117         struct rdma_cm_id       id;
118
119         struct rdma_bind_list   *bind_list;
120         struct hlist_node       node;
121         struct list_head        list; /* listen_any_list or cma_device.list */
122         struct list_head        listen_list; /* per device listens */
123         struct cma_device       *cma_dev;
124         struct list_head        mc_list;
125
126         int                     internal_id;
127         enum cma_state          state;
128         spinlock_t              lock;
129         struct mutex            qp_mutex;
130
131         struct completion       comp;
132         atomic_t                refcount;
133         wait_queue_head_t       wait_remove;
134         atomic_t                dev_remove;
135
136         int                     backlog;
137         int                     timeout_ms;
138         struct ib_sa_query      *query;
139         int                     query_id;
140         union {
141                 struct ib_cm_id *ib;
142                 struct iw_cm_id *iw;
143         } cm_id;
144
145         u32                     seq_num;
146         u32                     qkey;
147         u32                     qp_num;
148         u8                      srq;
149         u8                      tos;
150 };
151
152 struct cma_multicast {
153         struct rdma_id_private *id_priv;
154         union {
155                 struct ib_sa_multicast *ib;
156         } multicast;
157         struct list_head        list;
158         void                    *context;
159         struct sockaddr         addr;
160         u8                      pad[sizeof(struct sockaddr_in6) -
161                                     sizeof(struct sockaddr)];
162 };
163
164 struct cma_work {
165         struct work_struct      work;
166         struct rdma_id_private  *id;
167         enum cma_state          old_state;
168         enum cma_state          new_state;
169         struct rdma_cm_event    event;
170 };
171
172 union cma_ip_addr {
173         struct in6_addr ip6;
174         struct {
175                 __be32 pad[3];
176                 __be32 addr;
177         } ip4;
178 };
179
180 struct cma_hdr {
181         u8 cma_version;
182         u8 ip_version;  /* IP version: 7:4 */
183         __be16 port;
184         union cma_ip_addr src_addr;
185         union cma_ip_addr dst_addr;
186 };
187
188 struct sdp_hh {
189         u8 bsdh[16];
190         u8 sdp_version; /* Major version: 7:4 */
191         u8 ip_version;  /* IP version: 7:4 */
192         u8 sdp_specific1[10];
193         __be16 port;
194         __be16 sdp_specific2;
195         union cma_ip_addr src_addr;
196         union cma_ip_addr dst_addr;
197 };
198
199 struct sdp_hah {
200         u8 bsdh[16];
201         u8 sdp_version;
202 };
203
204 #define CMA_VERSION 0x00
205 #define SDP_MAJ_VERSION 0x2
206
207 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
208 {
209         unsigned long flags;
210         int ret;
211
212         spin_lock_irqsave(&id_priv->lock, flags);
213         ret = (id_priv->state == comp);
214         spin_unlock_irqrestore(&id_priv->lock, flags);
215         return ret;
216 }
217
218 static int cma_comp_exch(struct rdma_id_private *id_priv,
219                          enum cma_state comp, enum cma_state exch)
220 {
221         unsigned long flags;
222         int ret;
223
224         spin_lock_irqsave(&id_priv->lock, flags);
225         if ((ret = (id_priv->state == comp)))
226                 id_priv->state = exch;
227         spin_unlock_irqrestore(&id_priv->lock, flags);
228         return ret;
229 }
230
231 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
232                                enum cma_state exch)
233 {
234         unsigned long flags;
235         enum cma_state old;
236
237         spin_lock_irqsave(&id_priv->lock, flags);
238         old = id_priv->state;
239         id_priv->state = exch;
240         spin_unlock_irqrestore(&id_priv->lock, flags);
241         return old;
242 }
243
244 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
245 {
246         return hdr->ip_version >> 4;
247 }
248
249 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
250 {
251         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
252 }
253
254 static inline u8 sdp_get_majv(u8 sdp_version)
255 {
256         return sdp_version >> 4;
257 }
258
259 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
260 {
261         return hh->ip_version >> 4;
262 }
263
264 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
265 {
266         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
267 }
268
269 static inline int cma_is_ud_ps(enum rdma_port_space ps)
270 {
271         return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
272 }
273
274 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
275                               struct cma_device *cma_dev)
276 {
277         atomic_inc(&cma_dev->refcount);
278         id_priv->cma_dev = cma_dev;
279         id_priv->id.device = cma_dev->device;
280         list_add_tail(&id_priv->list, &cma_dev->id_list);
281 }
282
283 static inline void cma_deref_dev(struct cma_device *cma_dev)
284 {
285         if (atomic_dec_and_test(&cma_dev->refcount))
286                 complete(&cma_dev->comp);
287 }
288
289 static void cma_detach_from_dev(struct rdma_id_private *id_priv)
290 {
291         list_del(&id_priv->list);
292         cma_deref_dev(id_priv->cma_dev);
293         id_priv->cma_dev = NULL;
294 }
295
296 static int cma_set_qkey(struct ib_device *device, u8 port_num,
297                         enum rdma_port_space ps,
298                         struct rdma_dev_addr *dev_addr, u32 *qkey)
299 {
300         struct ib_sa_mcmember_rec rec;
301         int ret = 0;
302
303         switch (ps) {
304         case RDMA_PS_UDP:
305                 *qkey = RDMA_UDP_QKEY;
306                 break;
307         case RDMA_PS_IPOIB:
308                 ib_addr_get_mgid(dev_addr, &rec.mgid);
309                 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
310                 *qkey = be32_to_cpu(rec.qkey);
311                 break;
312         default:
313                 break;
314         }
315         return ret;
316 }
317
318 static int cma_acquire_dev(struct rdma_id_private *id_priv)
319 {
320         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
321         struct cma_device *cma_dev;
322         union ib_gid gid;
323         int ret = -ENODEV;
324
325         switch (rdma_node_get_transport(dev_addr->dev_type)) {
326         case RDMA_TRANSPORT_IB:
327                 ib_addr_get_sgid(dev_addr, &gid);
328                 break;
329         case RDMA_TRANSPORT_IWARP:
330                 iw_addr_get_sgid(dev_addr, &gid);
331                 break;
332         default:
333                 return -ENODEV;
334         }
335
336         list_for_each_entry(cma_dev, &dev_list, list) {
337                 ret = ib_find_cached_gid(cma_dev->device, &gid,
338                                          &id_priv->id.port_num, NULL);
339                 if (!ret) {
340                         ret = cma_set_qkey(cma_dev->device,
341                                            id_priv->id.port_num,
342                                            id_priv->id.ps, dev_addr,
343                                            &id_priv->qkey);
344                         if (!ret)
345                                 cma_attach_to_dev(id_priv, cma_dev);
346                         break;
347                 }
348         }
349         return ret;
350 }
351
352 static void cma_deref_id(struct rdma_id_private *id_priv)
353 {
354         if (atomic_dec_and_test(&id_priv->refcount))
355                 complete(&id_priv->comp);
356 }
357
358 static int cma_disable_remove(struct rdma_id_private *id_priv,
359                               enum cma_state state)
360 {
361         unsigned long flags;
362         int ret;
363
364         spin_lock_irqsave(&id_priv->lock, flags);
365         if (id_priv->state == state) {
366                 atomic_inc(&id_priv->dev_remove);
367                 ret = 0;
368         } else
369                 ret = -EINVAL;
370         spin_unlock_irqrestore(&id_priv->lock, flags);
371         return ret;
372 }
373
374 static void cma_enable_remove(struct rdma_id_private *id_priv)
375 {
376         if (atomic_dec_and_test(&id_priv->dev_remove))
377                 wake_up(&id_priv->wait_remove);
378 }
379
380 static int cma_has_cm_dev(struct rdma_id_private *id_priv)
381 {
382         return (id_priv->id.device && id_priv->cm_id.ib);
383 }
384
385 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
386                                   void *context, enum rdma_port_space ps)
387 {
388         struct rdma_id_private *id_priv;
389
390         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
391         if (!id_priv)
392                 return ERR_PTR(-ENOMEM);
393
394         id_priv->state = CMA_IDLE;
395         id_priv->id.context = context;
396         id_priv->id.event_handler = event_handler;
397         id_priv->id.ps = ps;
398         spin_lock_init(&id_priv->lock);
399         mutex_init(&id_priv->qp_mutex);
400         init_completion(&id_priv->comp);
401         atomic_set(&id_priv->refcount, 1);
402         init_waitqueue_head(&id_priv->wait_remove);
403         atomic_set(&id_priv->dev_remove, 0);
404         INIT_LIST_HEAD(&id_priv->listen_list);
405         INIT_LIST_HEAD(&id_priv->mc_list);
406         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
407
408         return &id_priv->id;
409 }
410 EXPORT_SYMBOL(rdma_create_id);
411
412 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
413 {
414         struct ib_qp_attr qp_attr;
415         int qp_attr_mask, ret;
416
417         qp_attr.qp_state = IB_QPS_INIT;
418         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
419         if (ret)
420                 return ret;
421
422         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
423         if (ret)
424                 return ret;
425
426         qp_attr.qp_state = IB_QPS_RTR;
427         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
428         if (ret)
429                 return ret;
430
431         qp_attr.qp_state = IB_QPS_RTS;
432         qp_attr.sq_psn = 0;
433         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
434
435         return ret;
436 }
437
438 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
439 {
440         struct ib_qp_attr qp_attr;
441         int qp_attr_mask, ret;
442
443         qp_attr.qp_state = IB_QPS_INIT;
444         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
445         if (ret)
446                 return ret;
447
448         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
449 }
450
451 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
452                    struct ib_qp_init_attr *qp_init_attr)
453 {
454         struct rdma_id_private *id_priv;
455         struct ib_qp *qp;
456         int ret;
457
458         id_priv = container_of(id, struct rdma_id_private, id);
459         if (id->device != pd->device)
460                 return -EINVAL;
461
462         qp = ib_create_qp(pd, qp_init_attr);
463         if (IS_ERR(qp))
464                 return PTR_ERR(qp);
465
466         if (cma_is_ud_ps(id_priv->id.ps))
467                 ret = cma_init_ud_qp(id_priv, qp);
468         else
469                 ret = cma_init_conn_qp(id_priv, qp);
470         if (ret)
471                 goto err;
472
473         id->qp = qp;
474         id_priv->qp_num = qp->qp_num;
475         id_priv->srq = (qp->srq != NULL);
476         return 0;
477 err:
478         ib_destroy_qp(qp);
479         return ret;
480 }
481 EXPORT_SYMBOL(rdma_create_qp);
482
483 void rdma_destroy_qp(struct rdma_cm_id *id)
484 {
485         struct rdma_id_private *id_priv;
486
487         id_priv = container_of(id, struct rdma_id_private, id);
488         mutex_lock(&id_priv->qp_mutex);
489         ib_destroy_qp(id_priv->id.qp);
490         id_priv->id.qp = NULL;
491         mutex_unlock(&id_priv->qp_mutex);
492 }
493 EXPORT_SYMBOL(rdma_destroy_qp);
494
495 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
496                              struct rdma_conn_param *conn_param)
497 {
498         struct ib_qp_attr qp_attr;
499         int qp_attr_mask, ret;
500
501         mutex_lock(&id_priv->qp_mutex);
502         if (!id_priv->id.qp) {
503                 ret = 0;
504                 goto out;
505         }
506
507         /* Need to update QP attributes from default values. */
508         qp_attr.qp_state = IB_QPS_INIT;
509         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
510         if (ret)
511                 goto out;
512
513         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
514         if (ret)
515                 goto out;
516
517         qp_attr.qp_state = IB_QPS_RTR;
518         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
519         if (ret)
520                 goto out;
521
522         if (conn_param)
523                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
524         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
525 out:
526         mutex_unlock(&id_priv->qp_mutex);
527         return ret;
528 }
529
530 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
531                              struct rdma_conn_param *conn_param)
532 {
533         struct ib_qp_attr qp_attr;
534         int qp_attr_mask, ret;
535
536         mutex_lock(&id_priv->qp_mutex);
537         if (!id_priv->id.qp) {
538                 ret = 0;
539                 goto out;
540         }
541
542         qp_attr.qp_state = IB_QPS_RTS;
543         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
544         if (ret)
545                 goto out;
546
547         if (conn_param)
548                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
549         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
550 out:
551         mutex_unlock(&id_priv->qp_mutex);
552         return ret;
553 }
554
555 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
556 {
557         struct ib_qp_attr qp_attr;
558         int ret;
559
560         mutex_lock(&id_priv->qp_mutex);
561         if (!id_priv->id.qp) {
562                 ret = 0;
563                 goto out;
564         }
565
566         qp_attr.qp_state = IB_QPS_ERR;
567         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
568 out:
569         mutex_unlock(&id_priv->qp_mutex);
570         return ret;
571 }
572
573 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
574                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
575 {
576         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
577         int ret;
578
579         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
580                                   ib_addr_get_pkey(dev_addr),
581                                   &qp_attr->pkey_index);
582         if (ret)
583                 return ret;
584
585         qp_attr->port_num = id_priv->id.port_num;
586         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
587
588         if (cma_is_ud_ps(id_priv->id.ps)) {
589                 qp_attr->qkey = id_priv->qkey;
590                 *qp_attr_mask |= IB_QP_QKEY;
591         } else {
592                 qp_attr->qp_access_flags = 0;
593                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
594         }
595         return 0;
596 }
597
598 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
599                        int *qp_attr_mask)
600 {
601         struct rdma_id_private *id_priv;
602         int ret = 0;
603
604         id_priv = container_of(id, struct rdma_id_private, id);
605         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
606         case RDMA_TRANSPORT_IB:
607                 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
608                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
609                 else
610                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
611                                                  qp_attr_mask);
612                 if (qp_attr->qp_state == IB_QPS_RTR)
613                         qp_attr->rq_psn = id_priv->seq_num;
614                 break;
615         case RDMA_TRANSPORT_IWARP:
616                 if (!id_priv->cm_id.iw) {
617                         qp_attr->qp_access_flags = 0;
618                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
619                 } else
620                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
621                                                  qp_attr_mask);
622                 break;
623         default:
624                 ret = -ENOSYS;
625                 break;
626         }
627
628         return ret;
629 }
630 EXPORT_SYMBOL(rdma_init_qp_attr);
631
632 static inline int cma_zero_addr(struct sockaddr *addr)
633 {
634         struct in6_addr *ip6;
635
636         if (addr->sa_family == AF_INET)
637                 return ipv4_is_zeronet(
638                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
639         else {
640                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
641                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
642                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
643         }
644 }
645
646 static inline int cma_loopback_addr(struct sockaddr *addr)
647 {
648         return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
649 }
650
651 static inline int cma_any_addr(struct sockaddr *addr)
652 {
653         return cma_zero_addr(addr) || cma_loopback_addr(addr);
654 }
655
656 static inline __be16 cma_port(struct sockaddr *addr)
657 {
658         if (addr->sa_family == AF_INET)
659                 return ((struct sockaddr_in *) addr)->sin_port;
660         else
661                 return ((struct sockaddr_in6 *) addr)->sin6_port;
662 }
663
664 static inline int cma_any_port(struct sockaddr *addr)
665 {
666         return !cma_port(addr);
667 }
668
669 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
670                             u8 *ip_ver, __be16 *port,
671                             union cma_ip_addr **src, union cma_ip_addr **dst)
672 {
673         switch (ps) {
674         case RDMA_PS_SDP:
675                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
676                     SDP_MAJ_VERSION)
677                         return -EINVAL;
678
679                 *ip_ver = sdp_get_ip_ver(hdr);
680                 *port   = ((struct sdp_hh *) hdr)->port;
681                 *src    = &((struct sdp_hh *) hdr)->src_addr;
682                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
683                 break;
684         default:
685                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
686                         return -EINVAL;
687
688                 *ip_ver = cma_get_ip_ver(hdr);
689                 *port   = ((struct cma_hdr *) hdr)->port;
690                 *src    = &((struct cma_hdr *) hdr)->src_addr;
691                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
692                 break;
693         }
694
695         if (*ip_ver != 4 && *ip_ver != 6)
696                 return -EINVAL;
697         return 0;
698 }
699
700 static void cma_save_net_info(struct rdma_addr *addr,
701                               struct rdma_addr *listen_addr,
702                               u8 ip_ver, __be16 port,
703                               union cma_ip_addr *src, union cma_ip_addr *dst)
704 {
705         struct sockaddr_in *listen4, *ip4;
706         struct sockaddr_in6 *listen6, *ip6;
707
708         switch (ip_ver) {
709         case 4:
710                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
711                 ip4 = (struct sockaddr_in *) &addr->src_addr;
712                 ip4->sin_family = listen4->sin_family;
713                 ip4->sin_addr.s_addr = dst->ip4.addr;
714                 ip4->sin_port = listen4->sin_port;
715
716                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
717                 ip4->sin_family = listen4->sin_family;
718                 ip4->sin_addr.s_addr = src->ip4.addr;
719                 ip4->sin_port = port;
720                 break;
721         case 6:
722                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
723                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
724                 ip6->sin6_family = listen6->sin6_family;
725                 ip6->sin6_addr = dst->ip6;
726                 ip6->sin6_port = listen6->sin6_port;
727
728                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
729                 ip6->sin6_family = listen6->sin6_family;
730                 ip6->sin6_addr = src->ip6;
731                 ip6->sin6_port = port;
732                 break;
733         default:
734                 break;
735         }
736 }
737
738 static inline int cma_user_data_offset(enum rdma_port_space ps)
739 {
740         switch (ps) {
741         case RDMA_PS_SDP:
742                 return 0;
743         default:
744                 return sizeof(struct cma_hdr);
745         }
746 }
747
748 static void cma_cancel_route(struct rdma_id_private *id_priv)
749 {
750         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
751         case RDMA_TRANSPORT_IB:
752                 if (id_priv->query)
753                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
754                 break;
755         default:
756                 break;
757         }
758 }
759
760 static void cma_cancel_listens(struct rdma_id_private *id_priv)
761 {
762         struct rdma_id_private *dev_id_priv;
763
764         /*
765          * Remove from listen_any_list to prevent added devices from spawning
766          * additional listen requests.
767          */
768         mutex_lock(&lock);
769         list_del(&id_priv->list);
770
771         while (!list_empty(&id_priv->listen_list)) {
772                 dev_id_priv = list_entry(id_priv->listen_list.next,
773                                          struct rdma_id_private, listen_list);
774                 /* sync with device removal to avoid duplicate destruction */
775                 list_del_init(&dev_id_priv->list);
776                 list_del(&dev_id_priv->listen_list);
777                 mutex_unlock(&lock);
778
779                 rdma_destroy_id(&dev_id_priv->id);
780                 mutex_lock(&lock);
781         }
782         mutex_unlock(&lock);
783 }
784
785 static void cma_cancel_operation(struct rdma_id_private *id_priv,
786                                  enum cma_state state)
787 {
788         switch (state) {
789         case CMA_ADDR_QUERY:
790                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
791                 break;
792         case CMA_ROUTE_QUERY:
793                 cma_cancel_route(id_priv);
794                 break;
795         case CMA_LISTEN:
796                 if (cma_any_addr(&id_priv->id.route.addr.src_addr) &&
797                     !id_priv->cma_dev)
798                         cma_cancel_listens(id_priv);
799                 break;
800         default:
801                 break;
802         }
803 }
804
805 static void cma_release_port(struct rdma_id_private *id_priv)
806 {
807         struct rdma_bind_list *bind_list = id_priv->bind_list;
808
809         if (!bind_list)
810                 return;
811
812         mutex_lock(&lock);
813         hlist_del(&id_priv->node);
814         if (hlist_empty(&bind_list->owners)) {
815                 idr_remove(bind_list->ps, bind_list->port);
816                 kfree(bind_list);
817         }
818         mutex_unlock(&lock);
819 }
820
821 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
822 {
823         struct cma_multicast *mc;
824
825         while (!list_empty(&id_priv->mc_list)) {
826                 mc = container_of(id_priv->mc_list.next,
827                                   struct cma_multicast, list);
828                 list_del(&mc->list);
829                 ib_sa_free_multicast(mc->multicast.ib);
830                 kfree(mc);
831         }
832 }
833
834 void rdma_destroy_id(struct rdma_cm_id *id)
835 {
836         struct rdma_id_private *id_priv;
837         enum cma_state state;
838
839         id_priv = container_of(id, struct rdma_id_private, id);
840         state = cma_exch(id_priv, CMA_DESTROYING);
841         cma_cancel_operation(id_priv, state);
842
843         mutex_lock(&lock);
844         if (id_priv->cma_dev) {
845                 mutex_unlock(&lock);
846                 switch (rdma_node_get_transport(id->device->node_type)) {
847                 case RDMA_TRANSPORT_IB:
848                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
849                                 ib_destroy_cm_id(id_priv->cm_id.ib);
850                         break;
851                 case RDMA_TRANSPORT_IWARP:
852                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
853                                 iw_destroy_cm_id(id_priv->cm_id.iw);
854                         break;
855                 default:
856                         break;
857                 }
858                 cma_leave_mc_groups(id_priv);
859                 mutex_lock(&lock);
860                 cma_detach_from_dev(id_priv);
861         }
862         mutex_unlock(&lock);
863
864         cma_release_port(id_priv);
865         cma_deref_id(id_priv);
866         wait_for_completion(&id_priv->comp);
867
868         if (id_priv->internal_id)
869                 cma_deref_id(id_priv->id.context);
870
871         kfree(id_priv->id.route.path_rec);
872         kfree(id_priv);
873 }
874 EXPORT_SYMBOL(rdma_destroy_id);
875
876 static int cma_rep_recv(struct rdma_id_private *id_priv)
877 {
878         int ret;
879
880         ret = cma_modify_qp_rtr(id_priv, NULL);
881         if (ret)
882                 goto reject;
883
884         ret = cma_modify_qp_rts(id_priv, NULL);
885         if (ret)
886                 goto reject;
887
888         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
889         if (ret)
890                 goto reject;
891
892         return 0;
893 reject:
894         cma_modify_qp_err(id_priv);
895         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
896                        NULL, 0, NULL, 0);
897         return ret;
898 }
899
900 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
901 {
902         if (id_priv->id.ps == RDMA_PS_SDP &&
903             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
904             SDP_MAJ_VERSION)
905                 return -EINVAL;
906
907         return 0;
908 }
909
910 static void cma_set_rep_event_data(struct rdma_cm_event *event,
911                                    struct ib_cm_rep_event_param *rep_data,
912                                    void *private_data)
913 {
914         event->param.conn.private_data = private_data;
915         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
916         event->param.conn.responder_resources = rep_data->responder_resources;
917         event->param.conn.initiator_depth = rep_data->initiator_depth;
918         event->param.conn.flow_control = rep_data->flow_control;
919         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
920         event->param.conn.srq = rep_data->srq;
921         event->param.conn.qp_num = rep_data->remote_qpn;
922 }
923
924 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
925 {
926         struct rdma_id_private *id_priv = cm_id->context;
927         struct rdma_cm_event event;
928         int ret = 0;
929
930         if (cma_disable_remove(id_priv, CMA_CONNECT))
931                 return 0;
932
933         memset(&event, 0, sizeof event);
934         switch (ib_event->event) {
935         case IB_CM_REQ_ERROR:
936         case IB_CM_REP_ERROR:
937                 event.event = RDMA_CM_EVENT_UNREACHABLE;
938                 event.status = -ETIMEDOUT;
939                 break;
940         case IB_CM_REP_RECEIVED:
941                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
942                 if (event.status)
943                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
944                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
945                         event.status = cma_rep_recv(id_priv);
946                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
947                                                      RDMA_CM_EVENT_ESTABLISHED;
948                 } else
949                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
950                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
951                                        ib_event->private_data);
952                 break;
953         case IB_CM_RTU_RECEIVED:
954         case IB_CM_USER_ESTABLISHED:
955                 event.event = RDMA_CM_EVENT_ESTABLISHED;
956                 break;
957         case IB_CM_DREQ_ERROR:
958                 event.status = -ETIMEDOUT; /* fall through */
959         case IB_CM_DREQ_RECEIVED:
960         case IB_CM_DREP_RECEIVED:
961                 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
962                         goto out;
963                 event.event = RDMA_CM_EVENT_DISCONNECTED;
964                 break;
965         case IB_CM_TIMEWAIT_EXIT:
966         case IB_CM_MRA_RECEIVED:
967                 /* ignore event */
968                 goto out;
969         case IB_CM_REJ_RECEIVED:
970                 cma_modify_qp_err(id_priv);
971                 event.status = ib_event->param.rej_rcvd.reason;
972                 event.event = RDMA_CM_EVENT_REJECTED;
973                 event.param.conn.private_data = ib_event->private_data;
974                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
975                 break;
976         default:
977                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
978                        ib_event->event);
979                 goto out;
980         }
981
982         ret = id_priv->id.event_handler(&id_priv->id, &event);
983         if (ret) {
984                 /* Destroy the CM ID by returning a non-zero value. */
985                 id_priv->cm_id.ib = NULL;
986                 cma_exch(id_priv, CMA_DESTROYING);
987                 cma_enable_remove(id_priv);
988                 rdma_destroy_id(&id_priv->id);
989                 return ret;
990         }
991 out:
992         cma_enable_remove(id_priv);
993         return ret;
994 }
995
996 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
997                                                struct ib_cm_event *ib_event)
998 {
999         struct rdma_id_private *id_priv;
1000         struct rdma_cm_id *id;
1001         struct rdma_route *rt;
1002         union cma_ip_addr *src, *dst;
1003         __be16 port;
1004         u8 ip_ver;
1005         int ret;
1006
1007         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1008                              &ip_ver, &port, &src, &dst))
1009                 goto err;
1010
1011         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1012                             listen_id->ps);
1013         if (IS_ERR(id))
1014                 goto err;
1015
1016         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1017                           ip_ver, port, src, dst);
1018
1019         rt = &id->route;
1020         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1021         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1022                                GFP_KERNEL);
1023         if (!rt->path_rec)
1024                 goto destroy_id;
1025
1026         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1027         if (rt->num_paths == 2)
1028                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1029
1030         ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1031         ret = rdma_translate_ip(&id->route.addr.src_addr,
1032                                 &id->route.addr.dev_addr);
1033         if (ret)
1034                 goto destroy_id;
1035
1036         id_priv = container_of(id, struct rdma_id_private, id);
1037         id_priv->state = CMA_CONNECT;
1038         return id_priv;
1039
1040 destroy_id:
1041         rdma_destroy_id(id);
1042 err:
1043         return NULL;
1044 }
1045
1046 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1047                                               struct ib_cm_event *ib_event)
1048 {
1049         struct rdma_id_private *id_priv;
1050         struct rdma_cm_id *id;
1051         union cma_ip_addr *src, *dst;
1052         __be16 port;
1053         u8 ip_ver;
1054         int ret;
1055
1056         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1057                             listen_id->ps);
1058         if (IS_ERR(id))
1059                 return NULL;
1060
1061
1062         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1063                              &ip_ver, &port, &src, &dst))
1064                 goto err;
1065
1066         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1067                           ip_ver, port, src, dst);
1068
1069         ret = rdma_translate_ip(&id->route.addr.src_addr,
1070                                 &id->route.addr.dev_addr);
1071         if (ret)
1072                 goto err;
1073
1074         id_priv = container_of(id, struct rdma_id_private, id);
1075         id_priv->state = CMA_CONNECT;
1076         return id_priv;
1077 err:
1078         rdma_destroy_id(id);
1079         return NULL;
1080 }
1081
1082 static void cma_set_req_event_data(struct rdma_cm_event *event,
1083                                    struct ib_cm_req_event_param *req_data,
1084                                    void *private_data, int offset)
1085 {
1086         event->param.conn.private_data = private_data + offset;
1087         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1088         event->param.conn.responder_resources = req_data->responder_resources;
1089         event->param.conn.initiator_depth = req_data->initiator_depth;
1090         event->param.conn.flow_control = req_data->flow_control;
1091         event->param.conn.retry_count = req_data->retry_count;
1092         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1093         event->param.conn.srq = req_data->srq;
1094         event->param.conn.qp_num = req_data->remote_qpn;
1095 }
1096
1097 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1098 {
1099         struct rdma_id_private *listen_id, *conn_id;
1100         struct rdma_cm_event event;
1101         int offset, ret;
1102
1103         listen_id = cm_id->context;
1104         if (cma_disable_remove(listen_id, CMA_LISTEN))
1105                 return -ECONNABORTED;
1106
1107         memset(&event, 0, sizeof event);
1108         offset = cma_user_data_offset(listen_id->id.ps);
1109         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1110         if (cma_is_ud_ps(listen_id->id.ps)) {
1111                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1112                 event.param.ud.private_data = ib_event->private_data + offset;
1113                 event.param.ud.private_data_len =
1114                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1115         } else {
1116                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1117                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1118                                        ib_event->private_data, offset);
1119         }
1120         if (!conn_id) {
1121                 ret = -ENOMEM;
1122                 goto out;
1123         }
1124
1125         atomic_inc(&conn_id->dev_remove);
1126         mutex_lock(&lock);
1127         ret = cma_acquire_dev(conn_id);
1128         mutex_unlock(&lock);
1129         if (ret)
1130                 goto release_conn_id;
1131
1132         conn_id->cm_id.ib = cm_id;
1133         cm_id->context = conn_id;
1134         cm_id->cm_handler = cma_ib_handler;
1135
1136         ret = conn_id->id.event_handler(&conn_id->id, &event);
1137         if (!ret) {
1138                 /*
1139                  * Acquire mutex to prevent user executing rdma_destroy_id()
1140                  * while we're accessing the cm_id.
1141                  */
1142                 mutex_lock(&lock);
1143                 if (cma_comp(conn_id, CMA_CONNECT) &&
1144                     !cma_is_ud_ps(conn_id->id.ps))
1145                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1146                 mutex_unlock(&lock);
1147                 cma_enable_remove(conn_id);
1148                 goto out;
1149         }
1150
1151         /* Destroy the CM ID by returning a non-zero value. */
1152         conn_id->cm_id.ib = NULL;
1153
1154 release_conn_id:
1155         cma_exch(conn_id, CMA_DESTROYING);
1156         cma_enable_remove(conn_id);
1157         rdma_destroy_id(&conn_id->id);
1158
1159 out:
1160         cma_enable_remove(listen_id);
1161         return ret;
1162 }
1163
1164 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1165 {
1166         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1167 }
1168
1169 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1170                                  struct ib_cm_compare_data *compare)
1171 {
1172         struct cma_hdr *cma_data, *cma_mask;
1173         struct sdp_hh *sdp_data, *sdp_mask;
1174         __be32 ip4_addr;
1175         struct in6_addr ip6_addr;
1176
1177         memset(compare, 0, sizeof *compare);
1178         cma_data = (void *) compare->data;
1179         cma_mask = (void *) compare->mask;
1180         sdp_data = (void *) compare->data;
1181         sdp_mask = (void *) compare->mask;
1182
1183         switch (addr->sa_family) {
1184         case AF_INET:
1185                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1186                 if (ps == RDMA_PS_SDP) {
1187                         sdp_set_ip_ver(sdp_data, 4);
1188                         sdp_set_ip_ver(sdp_mask, 0xF);
1189                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1190                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1191                 } else {
1192                         cma_set_ip_ver(cma_data, 4);
1193                         cma_set_ip_ver(cma_mask, 0xF);
1194                         cma_data->dst_addr.ip4.addr = ip4_addr;
1195                         cma_mask->dst_addr.ip4.addr = htonl(~0);
1196                 }
1197                 break;
1198         case AF_INET6:
1199                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1200                 if (ps == RDMA_PS_SDP) {
1201                         sdp_set_ip_ver(sdp_data, 6);
1202                         sdp_set_ip_ver(sdp_mask, 0xF);
1203                         sdp_data->dst_addr.ip6 = ip6_addr;
1204                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1205                                sizeof sdp_mask->dst_addr.ip6);
1206                 } else {
1207                         cma_set_ip_ver(cma_data, 6);
1208                         cma_set_ip_ver(cma_mask, 0xF);
1209                         cma_data->dst_addr.ip6 = ip6_addr;
1210                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1211                                sizeof cma_mask->dst_addr.ip6);
1212                 }
1213                 break;
1214         default:
1215                 break;
1216         }
1217 }
1218
1219 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1220 {
1221         struct rdma_id_private *id_priv = iw_id->context;
1222         struct rdma_cm_event event;
1223         struct sockaddr_in *sin;
1224         int ret = 0;
1225
1226         if (cma_disable_remove(id_priv, CMA_CONNECT))
1227                 return 0;
1228
1229         memset(&event, 0, sizeof event);
1230         switch (iw_event->event) {
1231         case IW_CM_EVENT_CLOSE:
1232                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1233                 break;
1234         case IW_CM_EVENT_CONNECT_REPLY:
1235                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1236                 *sin = iw_event->local_addr;
1237                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1238                 *sin = iw_event->remote_addr;
1239                 switch (iw_event->status) {
1240                 case 0:
1241                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1242                         break;
1243                 case -ECONNRESET:
1244                 case -ECONNREFUSED:
1245                         event.event = RDMA_CM_EVENT_REJECTED;
1246                         break;
1247                 case -ETIMEDOUT:
1248                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1249                         break;
1250                 default:
1251                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1252                         break;
1253                 }
1254                 break;
1255         case IW_CM_EVENT_ESTABLISHED:
1256                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1257                 break;
1258         default:
1259                 BUG_ON(1);
1260         }
1261
1262         event.status = iw_event->status;
1263         event.param.conn.private_data = iw_event->private_data;
1264         event.param.conn.private_data_len = iw_event->private_data_len;
1265         ret = id_priv->id.event_handler(&id_priv->id, &event);
1266         if (ret) {
1267                 /* Destroy the CM ID by returning a non-zero value. */
1268                 id_priv->cm_id.iw = NULL;
1269                 cma_exch(id_priv, CMA_DESTROYING);
1270                 cma_enable_remove(id_priv);
1271                 rdma_destroy_id(&id_priv->id);
1272                 return ret;
1273         }
1274
1275         cma_enable_remove(id_priv);
1276         return ret;
1277 }
1278
1279 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1280                                struct iw_cm_event *iw_event)
1281 {
1282         struct rdma_cm_id *new_cm_id;
1283         struct rdma_id_private *listen_id, *conn_id;
1284         struct sockaddr_in *sin;
1285         struct net_device *dev = NULL;
1286         struct rdma_cm_event event;
1287         int ret;
1288         struct ib_device_attr attr;
1289
1290         listen_id = cm_id->context;
1291         if (cma_disable_remove(listen_id, CMA_LISTEN))
1292                 return -ECONNABORTED;
1293
1294         /* Create a new RDMA id for the new IW CM ID */
1295         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1296                                    listen_id->id.context,
1297                                    RDMA_PS_TCP);
1298         if (IS_ERR(new_cm_id)) {
1299                 ret = -ENOMEM;
1300                 goto out;
1301         }
1302         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1303         atomic_inc(&conn_id->dev_remove);
1304         conn_id->state = CMA_CONNECT;
1305
1306         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1307         if (!dev) {
1308                 ret = -EADDRNOTAVAIL;
1309                 cma_enable_remove(conn_id);
1310                 rdma_destroy_id(new_cm_id);
1311                 goto out;
1312         }
1313         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1314         if (ret) {
1315                 cma_enable_remove(conn_id);
1316                 rdma_destroy_id(new_cm_id);
1317                 goto out;
1318         }
1319
1320         mutex_lock(&lock);
1321         ret = cma_acquire_dev(conn_id);
1322         mutex_unlock(&lock);
1323         if (ret) {
1324                 cma_enable_remove(conn_id);
1325                 rdma_destroy_id(new_cm_id);
1326                 goto out;
1327         }
1328
1329         conn_id->cm_id.iw = cm_id;
1330         cm_id->context = conn_id;
1331         cm_id->cm_handler = cma_iw_handler;
1332
1333         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1334         *sin = iw_event->local_addr;
1335         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1336         *sin = iw_event->remote_addr;
1337
1338         ret = ib_query_device(conn_id->id.device, &attr);
1339         if (ret) {
1340                 cma_enable_remove(conn_id);
1341                 rdma_destroy_id(new_cm_id);
1342                 goto out;
1343         }
1344
1345         memset(&event, 0, sizeof event);
1346         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1347         event.param.conn.private_data = iw_event->private_data;
1348         event.param.conn.private_data_len = iw_event->private_data_len;
1349         event.param.conn.initiator_depth = attr.max_qp_init_rd_atom;
1350         event.param.conn.responder_resources = attr.max_qp_rd_atom;
1351         ret = conn_id->id.event_handler(&conn_id->id, &event);
1352         if (ret) {
1353                 /* User wants to destroy the CM ID */
1354                 conn_id->cm_id.iw = NULL;
1355                 cma_exch(conn_id, CMA_DESTROYING);
1356                 cma_enable_remove(conn_id);
1357                 rdma_destroy_id(&conn_id->id);
1358         }
1359
1360 out:
1361         if (dev)
1362                 dev_put(dev);
1363         cma_enable_remove(listen_id);
1364         return ret;
1365 }
1366
1367 static int cma_ib_listen(struct rdma_id_private *id_priv)
1368 {
1369         struct ib_cm_compare_data compare_data;
1370         struct sockaddr *addr;
1371         __be64 svc_id;
1372         int ret;
1373
1374         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1375                                             id_priv);
1376         if (IS_ERR(id_priv->cm_id.ib))
1377                 return PTR_ERR(id_priv->cm_id.ib);
1378
1379         addr = &id_priv->id.route.addr.src_addr;
1380         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1381         if (cma_any_addr(addr))
1382                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1383         else {
1384                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1385                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1386         }
1387
1388         if (ret) {
1389                 ib_destroy_cm_id(id_priv->cm_id.ib);
1390                 id_priv->cm_id.ib = NULL;
1391         }
1392
1393         return ret;
1394 }
1395
1396 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1397 {
1398         int ret;
1399         struct sockaddr_in *sin;
1400
1401         id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1402                                             iw_conn_req_handler,
1403                                             id_priv);
1404         if (IS_ERR(id_priv->cm_id.iw))
1405                 return PTR_ERR(id_priv->cm_id.iw);
1406
1407         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1408         id_priv->cm_id.iw->local_addr = *sin;
1409
1410         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1411
1412         if (ret) {
1413                 iw_destroy_cm_id(id_priv->cm_id.iw);
1414                 id_priv->cm_id.iw = NULL;
1415         }
1416
1417         return ret;
1418 }
1419
1420 static int cma_listen_handler(struct rdma_cm_id *id,
1421                               struct rdma_cm_event *event)
1422 {
1423         struct rdma_id_private *id_priv = id->context;
1424
1425         id->context = id_priv->id.context;
1426         id->event_handler = id_priv->id.event_handler;
1427         return id_priv->id.event_handler(id, event);
1428 }
1429
1430 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1431                               struct cma_device *cma_dev)
1432 {
1433         struct rdma_id_private *dev_id_priv;
1434         struct rdma_cm_id *id;
1435         int ret;
1436
1437         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1438         if (IS_ERR(id))
1439                 return;
1440
1441         dev_id_priv = container_of(id, struct rdma_id_private, id);
1442
1443         dev_id_priv->state = CMA_ADDR_BOUND;
1444         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1445                ip_addr_size(&id_priv->id.route.addr.src_addr));
1446
1447         cma_attach_to_dev(dev_id_priv, cma_dev);
1448         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1449         atomic_inc(&id_priv->refcount);
1450         dev_id_priv->internal_id = 1;
1451
1452         ret = rdma_listen(id, id_priv->backlog);
1453         if (ret)
1454                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1455                        "listening on device %s\n", ret, cma_dev->device->name);
1456 }
1457
1458 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1459 {
1460         struct cma_device *cma_dev;
1461
1462         mutex_lock(&lock);
1463         list_add_tail(&id_priv->list, &listen_any_list);
1464         list_for_each_entry(cma_dev, &dev_list, list)
1465                 cma_listen_on_dev(id_priv, cma_dev);
1466         mutex_unlock(&lock);
1467 }
1468
1469 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1470 {
1471         struct sockaddr_in addr_in;
1472
1473         memset(&addr_in, 0, sizeof addr_in);
1474         addr_in.sin_family = af;
1475         return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1476 }
1477
1478 int rdma_listen(struct rdma_cm_id *id, int backlog)
1479 {
1480         struct rdma_id_private *id_priv;
1481         int ret;
1482
1483         id_priv = container_of(id, struct rdma_id_private, id);
1484         if (id_priv->state == CMA_IDLE) {
1485                 ret = cma_bind_any(id, AF_INET);
1486                 if (ret)
1487                         return ret;
1488         }
1489
1490         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1491                 return -EINVAL;
1492
1493         id_priv->backlog = backlog;
1494         if (id->device) {
1495                 switch (rdma_node_get_transport(id->device->node_type)) {
1496                 case RDMA_TRANSPORT_IB:
1497                         ret = cma_ib_listen(id_priv);
1498                         if (ret)
1499                                 goto err;
1500                         break;
1501                 case RDMA_TRANSPORT_IWARP:
1502                         ret = cma_iw_listen(id_priv, backlog);
1503                         if (ret)
1504                                 goto err;
1505                         break;
1506                 default:
1507                         ret = -ENOSYS;
1508                         goto err;
1509                 }
1510         } else
1511                 cma_listen_on_all(id_priv);
1512
1513         return 0;
1514 err:
1515         id_priv->backlog = 0;
1516         cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1517         return ret;
1518 }
1519 EXPORT_SYMBOL(rdma_listen);
1520
1521 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1522 {
1523         struct rdma_id_private *id_priv;
1524
1525         id_priv = container_of(id, struct rdma_id_private, id);
1526         id_priv->tos = (u8) tos;
1527 }
1528 EXPORT_SYMBOL(rdma_set_service_type);
1529
1530 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1531                               void *context)
1532 {
1533         struct cma_work *work = context;
1534         struct rdma_route *route;
1535
1536         route = &work->id->id.route;
1537
1538         if (!status) {
1539                 route->num_paths = 1;
1540                 *route->path_rec = *path_rec;
1541         } else {
1542                 work->old_state = CMA_ROUTE_QUERY;
1543                 work->new_state = CMA_ADDR_RESOLVED;
1544                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1545                 work->event.status = status;
1546         }
1547
1548         queue_work(cma_wq, &work->work);
1549 }
1550
1551 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1552                               struct cma_work *work)
1553 {
1554         struct rdma_addr *addr = &id_priv->id.route.addr;
1555         struct ib_sa_path_rec path_rec;
1556         ib_sa_comp_mask comp_mask;
1557         struct sockaddr_in6 *sin6;
1558
1559         memset(&path_rec, 0, sizeof path_rec);
1560         ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1561         ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1562         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1563         path_rec.numb_path = 1;
1564         path_rec.reversible = 1;
1565         path_rec.service_id = cma_get_service_id(id_priv->id.ps, &addr->dst_addr);
1566
1567         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1568                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1569                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1570
1571         if (addr->src_addr.sa_family == AF_INET) {
1572                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1573                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1574         } else {
1575                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1576                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1577                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1578         }
1579
1580         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1581                                                id_priv->id.port_num, &path_rec,
1582                                                comp_mask, timeout_ms,
1583                                                GFP_KERNEL, cma_query_handler,
1584                                                work, &id_priv->query);
1585
1586         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1587 }
1588
1589 static void cma_work_handler(struct work_struct *_work)
1590 {
1591         struct cma_work *work = container_of(_work, struct cma_work, work);
1592         struct rdma_id_private *id_priv = work->id;
1593         int destroy = 0;
1594
1595         atomic_inc(&id_priv->dev_remove);
1596         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1597                 goto out;
1598
1599         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1600                 cma_exch(id_priv, CMA_DESTROYING);
1601                 destroy = 1;
1602         }
1603 out:
1604         cma_enable_remove(id_priv);
1605         cma_deref_id(id_priv);
1606         if (destroy)
1607                 rdma_destroy_id(&id_priv->id);
1608         kfree(work);
1609 }
1610
1611 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1612 {
1613         struct rdma_route *route = &id_priv->id.route;
1614         struct cma_work *work;
1615         int ret;
1616
1617         work = kzalloc(sizeof *work, GFP_KERNEL);
1618         if (!work)
1619                 return -ENOMEM;
1620
1621         work->id = id_priv;
1622         INIT_WORK(&work->work, cma_work_handler);
1623         work->old_state = CMA_ROUTE_QUERY;
1624         work->new_state = CMA_ROUTE_RESOLVED;
1625         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1626
1627         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1628         if (!route->path_rec) {
1629                 ret = -ENOMEM;
1630                 goto err1;
1631         }
1632
1633         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1634         if (ret)
1635                 goto err2;
1636
1637         return 0;
1638 err2:
1639         kfree(route->path_rec);
1640         route->path_rec = NULL;
1641 err1:
1642         kfree(work);
1643         return ret;
1644 }
1645
1646 int rdma_set_ib_paths(struct rdma_cm_id *id,
1647                       struct ib_sa_path_rec *path_rec, int num_paths)
1648 {
1649         struct rdma_id_private *id_priv;
1650         int ret;
1651
1652         id_priv = container_of(id, struct rdma_id_private, id);
1653         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1654                 return -EINVAL;
1655
1656         id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1657         if (!id->route.path_rec) {
1658                 ret = -ENOMEM;
1659                 goto err;
1660         }
1661
1662         memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1663         return 0;
1664 err:
1665         cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1666         return ret;
1667 }
1668 EXPORT_SYMBOL(rdma_set_ib_paths);
1669
1670 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1671 {
1672         struct cma_work *work;
1673
1674         work = kzalloc(sizeof *work, GFP_KERNEL);
1675         if (!work)
1676                 return -ENOMEM;
1677
1678         work->id = id_priv;
1679         INIT_WORK(&work->work, cma_work_handler);
1680         work->old_state = CMA_ROUTE_QUERY;
1681         work->new_state = CMA_ROUTE_RESOLVED;
1682         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1683         queue_work(cma_wq, &work->work);
1684         return 0;
1685 }
1686
1687 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1688 {
1689         struct rdma_id_private *id_priv;
1690         int ret;
1691
1692         id_priv = container_of(id, struct rdma_id_private, id);
1693         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1694                 return -EINVAL;
1695
1696         atomic_inc(&id_priv->refcount);
1697         switch (rdma_node_get_transport(id->device->node_type)) {
1698         case RDMA_TRANSPORT_IB:
1699                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1700                 break;
1701         case RDMA_TRANSPORT_IWARP:
1702                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1703                 break;
1704         default:
1705                 ret = -ENOSYS;
1706                 break;
1707         }
1708         if (ret)
1709                 goto err;
1710
1711         return 0;
1712 err:
1713         cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1714         cma_deref_id(id_priv);
1715         return ret;
1716 }
1717 EXPORT_SYMBOL(rdma_resolve_route);
1718
1719 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1720 {
1721         struct cma_device *cma_dev;
1722         struct ib_port_attr port_attr;
1723         union ib_gid gid;
1724         u16 pkey;
1725         int ret;
1726         u8 p;
1727
1728         mutex_lock(&lock);
1729         if (list_empty(&dev_list)) {
1730                 ret = -ENODEV;
1731                 goto out;
1732         }
1733         list_for_each_entry(cma_dev, &dev_list, list)
1734                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1735                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1736                             port_attr.state == IB_PORT_ACTIVE)
1737                                 goto port_found;
1738
1739         p = 1;
1740         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1741
1742 port_found:
1743         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1744         if (ret)
1745                 goto out;
1746
1747         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1748         if (ret)
1749                 goto out;
1750
1751         ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1752         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1753         id_priv->id.port_num = p;
1754         cma_attach_to_dev(id_priv, cma_dev);
1755 out:
1756         mutex_unlock(&lock);
1757         return ret;
1758 }
1759
1760 static void addr_handler(int status, struct sockaddr *src_addr,
1761                          struct rdma_dev_addr *dev_addr, void *context)
1762 {
1763         struct rdma_id_private *id_priv = context;
1764         struct rdma_cm_event event;
1765
1766         memset(&event, 0, sizeof event);
1767         atomic_inc(&id_priv->dev_remove);
1768
1769         /*
1770          * Grab mutex to block rdma_destroy_id() from removing the device while
1771          * we're trying to acquire it.
1772          */
1773         mutex_lock(&lock);
1774         if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1775                 mutex_unlock(&lock);
1776                 goto out;
1777         }
1778
1779         if (!status && !id_priv->cma_dev)
1780                 status = cma_acquire_dev(id_priv);
1781         mutex_unlock(&lock);
1782
1783         if (status) {
1784                 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1785                         goto out;
1786                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1787                 event.status = status;
1788         } else {
1789                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1790                        ip_addr_size(src_addr));
1791                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1792         }
1793
1794         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1795                 cma_exch(id_priv, CMA_DESTROYING);
1796                 cma_enable_remove(id_priv);
1797                 cma_deref_id(id_priv);
1798                 rdma_destroy_id(&id_priv->id);
1799                 return;
1800         }
1801 out:
1802         cma_enable_remove(id_priv);
1803         cma_deref_id(id_priv);
1804 }
1805
1806 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1807 {
1808         struct cma_work *work;
1809         struct sockaddr_in *src_in, *dst_in;
1810         union ib_gid gid;
1811         int ret;
1812
1813         work = kzalloc(sizeof *work, GFP_KERNEL);
1814         if (!work)
1815                 return -ENOMEM;
1816
1817         if (!id_priv->cma_dev) {
1818                 ret = cma_bind_loopback(id_priv);
1819                 if (ret)
1820                         goto err;
1821         }
1822
1823         ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1824         ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
1825
1826         if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) {
1827                 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1828                 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1829                 src_in->sin_family = dst_in->sin_family;
1830                 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1831         }
1832
1833         work->id = id_priv;
1834         INIT_WORK(&work->work, cma_work_handler);
1835         work->old_state = CMA_ADDR_QUERY;
1836         work->new_state = CMA_ADDR_RESOLVED;
1837         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1838         queue_work(cma_wq, &work->work);
1839         return 0;
1840 err:
1841         kfree(work);
1842         return ret;
1843 }
1844
1845 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1846                          struct sockaddr *dst_addr)
1847 {
1848         if (src_addr && src_addr->sa_family)
1849                 return rdma_bind_addr(id, src_addr);
1850         else
1851                 return cma_bind_any(id, dst_addr->sa_family);
1852 }
1853
1854 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1855                       struct sockaddr *dst_addr, int timeout_ms)
1856 {
1857         struct rdma_id_private *id_priv;
1858         int ret;
1859
1860         id_priv = container_of(id, struct rdma_id_private, id);
1861         if (id_priv->state == CMA_IDLE) {
1862                 ret = cma_bind_addr(id, src_addr, dst_addr);
1863                 if (ret)
1864                         return ret;
1865         }
1866
1867         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1868                 return -EINVAL;
1869
1870         atomic_inc(&id_priv->refcount);
1871         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1872         if (cma_any_addr(dst_addr))
1873                 ret = cma_resolve_loopback(id_priv);
1874         else
1875                 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr,
1876                                       dst_addr, &id->route.addr.dev_addr,
1877                                       timeout_ms, addr_handler, id_priv);
1878         if (ret)
1879                 goto err;
1880
1881         return 0;
1882 err:
1883         cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1884         cma_deref_id(id_priv);
1885         return ret;
1886 }
1887 EXPORT_SYMBOL(rdma_resolve_addr);
1888
1889 static void cma_bind_port(struct rdma_bind_list *bind_list,
1890                           struct rdma_id_private *id_priv)
1891 {
1892         struct sockaddr_in *sin;
1893
1894         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1895         sin->sin_port = htons(bind_list->port);
1896         id_priv->bind_list = bind_list;
1897         hlist_add_head(&id_priv->node, &bind_list->owners);
1898 }
1899
1900 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1901                           unsigned short snum)
1902 {
1903         struct rdma_bind_list *bind_list;
1904         int port, ret;
1905
1906         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1907         if (!bind_list)
1908                 return -ENOMEM;
1909
1910         do {
1911                 ret = idr_get_new_above(ps, bind_list, snum, &port);
1912         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1913
1914         if (ret)
1915                 goto err1;
1916
1917         if (port != snum) {
1918                 ret = -EADDRNOTAVAIL;
1919                 goto err2;
1920         }
1921
1922         bind_list->ps = ps;
1923         bind_list->port = (unsigned short) port;
1924         cma_bind_port(bind_list, id_priv);
1925         return 0;
1926 err2:
1927         idr_remove(ps, port);
1928 err1:
1929         kfree(bind_list);
1930         return ret;
1931 }
1932
1933 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1934 {
1935         struct rdma_bind_list *bind_list;
1936         int port, ret, low, high;
1937
1938         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1939         if (!bind_list)
1940                 return -ENOMEM;
1941
1942 retry:
1943         /* FIXME: add proper port randomization per like inet_csk_get_port */
1944         do {
1945                 ret = idr_get_new_above(ps, bind_list, next_port, &port);
1946         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1947
1948         if (ret)
1949                 goto err1;
1950
1951         inet_get_local_port_range(&low, &high);
1952         if (port > high) {
1953                 if (next_port != low) {
1954                         idr_remove(ps, port);
1955                         next_port = low;
1956                         goto retry;
1957                 }
1958                 ret = -EADDRNOTAVAIL;
1959                 goto err2;
1960         }
1961
1962         if (port == high)
1963                 next_port = low;
1964         else
1965                 next_port = port + 1;
1966
1967         bind_list->ps = ps;
1968         bind_list->port = (unsigned short) port;
1969         cma_bind_port(bind_list, id_priv);
1970         return 0;
1971 err2:
1972         idr_remove(ps, port);
1973 err1:
1974         kfree(bind_list);
1975         return ret;
1976 }
1977
1978 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
1979 {
1980         struct rdma_id_private *cur_id;
1981         struct sockaddr_in *sin, *cur_sin;
1982         struct rdma_bind_list *bind_list;
1983         struct hlist_node *node;
1984         unsigned short snum;
1985
1986         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1987         snum = ntohs(sin->sin_port);
1988         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
1989                 return -EACCES;
1990
1991         bind_list = idr_find(ps, snum);
1992         if (!bind_list)
1993                 return cma_alloc_port(ps, id_priv, snum);
1994
1995         /*
1996          * We don't support binding to any address if anyone is bound to
1997          * a specific address on the same port.
1998          */
1999         if (cma_any_addr(&id_priv->id.route.addr.src_addr))
2000                 return -EADDRNOTAVAIL;
2001
2002         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2003                 if (cma_any_addr(&cur_id->id.route.addr.src_addr))
2004                         return -EADDRNOTAVAIL;
2005
2006                 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
2007                 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
2008                         return -EADDRINUSE;
2009         }
2010
2011         cma_bind_port(bind_list, id_priv);
2012         return 0;
2013 }
2014
2015 static int cma_get_port(struct rdma_id_private *id_priv)
2016 {
2017         struct idr *ps;
2018         int ret;
2019
2020         switch (id_priv->id.ps) {
2021         case RDMA_PS_SDP:
2022                 ps = &sdp_ps;
2023                 break;
2024         case RDMA_PS_TCP:
2025                 ps = &tcp_ps;
2026                 break;
2027         case RDMA_PS_UDP:
2028                 ps = &udp_ps;
2029                 break;
2030         case RDMA_PS_IPOIB:
2031                 ps = &ipoib_ps;
2032                 break;
2033         default:
2034                 return -EPROTONOSUPPORT;
2035         }
2036
2037         mutex_lock(&lock);
2038         if (cma_any_port(&id_priv->id.route.addr.src_addr))
2039                 ret = cma_alloc_any_port(ps, id_priv);
2040         else
2041                 ret = cma_use_port(ps, id_priv);
2042         mutex_unlock(&lock);
2043
2044         return ret;
2045 }
2046
2047 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2048 {
2049         struct rdma_id_private *id_priv;
2050         int ret;
2051
2052         if (addr->sa_family != AF_INET)
2053                 return -EAFNOSUPPORT;
2054
2055         id_priv = container_of(id, struct rdma_id_private, id);
2056         if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
2057                 return -EINVAL;
2058
2059         if (!cma_any_addr(addr)) {
2060                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2061                 if (ret)
2062                         goto err1;
2063
2064                 mutex_lock(&lock);
2065                 ret = cma_acquire_dev(id_priv);
2066                 mutex_unlock(&lock);
2067                 if (ret)
2068                         goto err1;
2069         }
2070
2071         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2072         ret = cma_get_port(id_priv);
2073         if (ret)
2074                 goto err2;
2075
2076         return 0;
2077 err2:
2078         if (!cma_any_addr(addr)) {
2079                 mutex_lock(&lock);
2080                 cma_detach_from_dev(id_priv);
2081                 mutex_unlock(&lock);
2082         }
2083 err1:
2084         cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2085         return ret;
2086 }
2087 EXPORT_SYMBOL(rdma_bind_addr);
2088
2089 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2090                           struct rdma_route *route)
2091 {
2092         struct sockaddr_in *src4, *dst4;
2093         struct cma_hdr *cma_hdr;
2094         struct sdp_hh *sdp_hdr;
2095
2096         src4 = (struct sockaddr_in *) &route->addr.src_addr;
2097         dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2098
2099         switch (ps) {
2100         case RDMA_PS_SDP:
2101                 sdp_hdr = hdr;
2102                 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2103                         return -EINVAL;
2104                 sdp_set_ip_ver(sdp_hdr, 4);
2105                 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2106                 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2107                 sdp_hdr->port = src4->sin_port;
2108                 break;
2109         default:
2110                 cma_hdr = hdr;
2111                 cma_hdr->cma_version = CMA_VERSION;
2112                 cma_set_ip_ver(cma_hdr, 4);
2113                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2114                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2115                 cma_hdr->port = src4->sin_port;
2116                 break;
2117         }
2118         return 0;
2119 }
2120
2121 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2122                                 struct ib_cm_event *ib_event)
2123 {
2124         struct rdma_id_private *id_priv = cm_id->context;
2125         struct rdma_cm_event event;
2126         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2127         int ret = 0;
2128
2129         if (cma_disable_remove(id_priv, CMA_CONNECT))
2130                 return 0;
2131
2132         memset(&event, 0, sizeof event);
2133         switch (ib_event->event) {
2134         case IB_CM_SIDR_REQ_ERROR:
2135                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2136                 event.status = -ETIMEDOUT;
2137                 break;
2138         case IB_CM_SIDR_REP_RECEIVED:
2139                 event.param.ud.private_data = ib_event->private_data;
2140                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2141                 if (rep->status != IB_SIDR_SUCCESS) {
2142                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2143                         event.status = ib_event->param.sidr_rep_rcvd.status;
2144                         break;
2145                 }
2146                 if (id_priv->qkey != rep->qkey) {
2147                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2148                         event.status = -EINVAL;
2149                         break;
2150                 }
2151                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2152                                      id_priv->id.route.path_rec,
2153                                      &event.param.ud.ah_attr);
2154                 event.param.ud.qp_num = rep->qpn;
2155                 event.param.ud.qkey = rep->qkey;
2156                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2157                 event.status = 0;
2158                 break;
2159         default:
2160                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2161                        ib_event->event);
2162                 goto out;
2163         }
2164
2165         ret = id_priv->id.event_handler(&id_priv->id, &event);
2166         if (ret) {
2167                 /* Destroy the CM ID by returning a non-zero value. */
2168                 id_priv->cm_id.ib = NULL;
2169                 cma_exch(id_priv, CMA_DESTROYING);
2170                 cma_enable_remove(id_priv);
2171                 rdma_destroy_id(&id_priv->id);
2172                 return ret;
2173         }
2174 out:
2175         cma_enable_remove(id_priv);
2176         return ret;
2177 }
2178
2179 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2180                               struct rdma_conn_param *conn_param)
2181 {
2182         struct ib_cm_sidr_req_param req;
2183         struct rdma_route *route;
2184         int ret;
2185
2186         req.private_data_len = sizeof(struct cma_hdr) +
2187                                conn_param->private_data_len;
2188         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2189         if (!req.private_data)
2190                 return -ENOMEM;
2191
2192         if (conn_param->private_data && conn_param->private_data_len)
2193                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2194                        conn_param->private_data, conn_param->private_data_len);
2195
2196         route = &id_priv->id.route;
2197         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2198         if (ret)
2199                 goto out;
2200
2201         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2202                                             cma_sidr_rep_handler, id_priv);
2203         if (IS_ERR(id_priv->cm_id.ib)) {
2204                 ret = PTR_ERR(id_priv->cm_id.ib);
2205                 goto out;
2206         }
2207
2208         req.path = route->path_rec;
2209         req.service_id = cma_get_service_id(id_priv->id.ps,
2210                                             &route->addr.dst_addr);
2211         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2212         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2213
2214         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2215         if (ret) {
2216                 ib_destroy_cm_id(id_priv->cm_id.ib);
2217                 id_priv->cm_id.ib = NULL;
2218         }
2219 out:
2220         kfree(req.private_data);
2221         return ret;
2222 }
2223
2224 static int cma_connect_ib(struct rdma_id_private *id_priv,
2225                           struct rdma_conn_param *conn_param)
2226 {
2227         struct ib_cm_req_param req;
2228         struct rdma_route *route;
2229         void *private_data;
2230         int offset, ret;
2231
2232         memset(&req, 0, sizeof req);
2233         offset = cma_user_data_offset(id_priv->id.ps);
2234         req.private_data_len = offset + conn_param->private_data_len;
2235         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2236         if (!private_data)
2237                 return -ENOMEM;
2238
2239         if (conn_param->private_data && conn_param->private_data_len)
2240                 memcpy(private_data + offset, conn_param->private_data,
2241                        conn_param->private_data_len);
2242
2243         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2244                                             id_priv);
2245         if (IS_ERR(id_priv->cm_id.ib)) {
2246                 ret = PTR_ERR(id_priv->cm_id.ib);
2247                 goto out;
2248         }
2249
2250         route = &id_priv->id.route;
2251         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2252         if (ret)
2253                 goto out;
2254         req.private_data = private_data;
2255
2256         req.primary_path = &route->path_rec[0];
2257         if (route->num_paths == 2)
2258                 req.alternate_path = &route->path_rec[1];
2259
2260         req.service_id = cma_get_service_id(id_priv->id.ps,
2261                                             &route->addr.dst_addr);
2262         req.qp_num = id_priv->qp_num;
2263         req.qp_type = IB_QPT_RC;
2264         req.starting_psn = id_priv->seq_num;
2265         req.responder_resources = conn_param->responder_resources;
2266         req.initiator_depth = conn_param->initiator_depth;
2267         req.flow_control = conn_param->flow_control;
2268         req.retry_count = conn_param->retry_count;
2269         req.rnr_retry_count = conn_param->rnr_retry_count;
2270         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2271         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2272         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2273         req.srq = id_priv->srq ? 1 : 0;
2274
2275         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2276 out:
2277         if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2278                 ib_destroy_cm_id(id_priv->cm_id.ib);
2279                 id_priv->cm_id.ib = NULL;
2280         }
2281
2282         kfree(private_data);
2283         return ret;
2284 }
2285
2286 static int cma_connect_iw(struct rdma_id_private *id_priv,
2287                           struct rdma_conn_param *conn_param)
2288 {
2289         struct iw_cm_id *cm_id;
2290         struct sockaddr_in* sin;
2291         int ret;
2292         struct iw_cm_conn_param iw_param;
2293
2294         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2295         if (IS_ERR(cm_id)) {
2296                 ret = PTR_ERR(cm_id);
2297                 goto out;
2298         }
2299
2300         id_priv->cm_id.iw = cm_id;
2301
2302         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2303         cm_id->local_addr = *sin;
2304
2305         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2306         cm_id->remote_addr = *sin;
2307
2308         ret = cma_modify_qp_rtr(id_priv, conn_param);
2309         if (ret)
2310                 goto out;
2311
2312         iw_param.ord = conn_param->initiator_depth;
2313         iw_param.ird = conn_param->responder_resources;
2314         iw_param.private_data = conn_param->private_data;
2315         iw_param.private_data_len = conn_param->private_data_len;
2316         if (id_priv->id.qp)
2317                 iw_param.qpn = id_priv->qp_num;
2318         else
2319                 iw_param.qpn = conn_param->qp_num;
2320         ret = iw_cm_connect(cm_id, &iw_param);
2321 out:
2322         if (ret && !IS_ERR(cm_id)) {
2323                 iw_destroy_cm_id(cm_id);
2324                 id_priv->cm_id.iw = NULL;
2325         }
2326         return ret;
2327 }
2328
2329 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2330 {
2331         struct rdma_id_private *id_priv;
2332         int ret;
2333
2334         id_priv = container_of(id, struct rdma_id_private, id);
2335         if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2336                 return -EINVAL;
2337
2338         if (!id->qp) {
2339                 id_priv->qp_num = conn_param->qp_num;
2340                 id_priv->srq = conn_param->srq;
2341         }
2342
2343         switch (rdma_node_get_transport(id->device->node_type)) {
2344         case RDMA_TRANSPORT_IB:
2345                 if (cma_is_ud_ps(id->ps))
2346                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2347                 else
2348                         ret = cma_connect_ib(id_priv, conn_param);
2349                 break;
2350         case RDMA_TRANSPORT_IWARP:
2351                 ret = cma_connect_iw(id_priv, conn_param);
2352                 break;
2353         default:
2354                 ret = -ENOSYS;
2355                 break;
2356         }
2357         if (ret)
2358                 goto err;
2359
2360         return 0;
2361 err:
2362         cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2363         return ret;
2364 }
2365 EXPORT_SYMBOL(rdma_connect);
2366
2367 static int cma_accept_ib(struct rdma_id_private *id_priv,
2368                          struct rdma_conn_param *conn_param)
2369 {
2370         struct ib_cm_rep_param rep;
2371         int ret;
2372
2373         ret = cma_modify_qp_rtr(id_priv, conn_param);
2374         if (ret)
2375                 goto out;
2376
2377         ret = cma_modify_qp_rts(id_priv, conn_param);
2378         if (ret)
2379                 goto out;
2380
2381         memset(&rep, 0, sizeof rep);
2382         rep.qp_num = id_priv->qp_num;
2383         rep.starting_psn = id_priv->seq_num;
2384         rep.private_data = conn_param->private_data;
2385         rep.private_data_len = conn_param->private_data_len;
2386         rep.responder_resources = conn_param->responder_resources;
2387         rep.initiator_depth = conn_param->initiator_depth;
2388         rep.failover_accepted = 0;
2389         rep.flow_control = conn_param->flow_control;
2390         rep.rnr_retry_count = conn_param->rnr_retry_count;
2391         rep.srq = id_priv->srq ? 1 : 0;
2392
2393         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2394 out:
2395         return ret;
2396 }
2397
2398 static int cma_accept_iw(struct rdma_id_private *id_priv,
2399                   struct rdma_conn_param *conn_param)
2400 {
2401         struct iw_cm_conn_param iw_param;
2402         int ret;
2403
2404         ret = cma_modify_qp_rtr(id_priv, conn_param);
2405         if (ret)
2406                 return ret;
2407
2408         iw_param.ord = conn_param->initiator_depth;
2409         iw_param.ird = conn_param->responder_resources;
2410         iw_param.private_data = conn_param->private_data;
2411         iw_param.private_data_len = conn_param->private_data_len;
2412         if (id_priv->id.qp) {
2413                 iw_param.qpn = id_priv->qp_num;
2414         } else
2415                 iw_param.qpn = conn_param->qp_num;
2416
2417         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2418 }
2419
2420 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2421                              enum ib_cm_sidr_status status,
2422                              const void *private_data, int private_data_len)
2423 {
2424         struct ib_cm_sidr_rep_param rep;
2425
2426         memset(&rep, 0, sizeof rep);
2427         rep.status = status;
2428         if (status == IB_SIDR_SUCCESS) {
2429                 rep.qp_num = id_priv->qp_num;
2430                 rep.qkey = id_priv->qkey;
2431         }
2432         rep.private_data = private_data;
2433         rep.private_data_len = private_data_len;
2434
2435         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2436 }
2437
2438 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2439 {
2440         struct rdma_id_private *id_priv;
2441         int ret;
2442
2443         id_priv = container_of(id, struct rdma_id_private, id);
2444         if (!cma_comp(id_priv, CMA_CONNECT))
2445                 return -EINVAL;
2446
2447         if (!id->qp && conn_param) {
2448                 id_priv->qp_num = conn_param->qp_num;
2449                 id_priv->srq = conn_param->srq;
2450         }
2451
2452         switch (rdma_node_get_transport(id->device->node_type)) {
2453         case RDMA_TRANSPORT_IB:
2454                 if (cma_is_ud_ps(id->ps))
2455                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2456                                                 conn_param->private_data,
2457                                                 conn_param->private_data_len);
2458                 else if (conn_param)
2459                         ret = cma_accept_ib(id_priv, conn_param);
2460                 else
2461                         ret = cma_rep_recv(id_priv);
2462                 break;
2463         case RDMA_TRANSPORT_IWARP:
2464                 ret = cma_accept_iw(id_priv, conn_param);
2465                 break;
2466         default:
2467                 ret = -ENOSYS;
2468                 break;
2469         }
2470
2471         if (ret)
2472                 goto reject;
2473
2474         return 0;
2475 reject:
2476         cma_modify_qp_err(id_priv);
2477         rdma_reject(id, NULL, 0);
2478         return ret;
2479 }
2480 EXPORT_SYMBOL(rdma_accept);
2481
2482 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2483 {
2484         struct rdma_id_private *id_priv;
2485         int ret;
2486
2487         id_priv = container_of(id, struct rdma_id_private, id);
2488         if (!cma_has_cm_dev(id_priv))
2489                 return -EINVAL;
2490
2491         switch (id->device->node_type) {
2492         case RDMA_NODE_IB_CA:
2493                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2494                 break;
2495         default:
2496                 ret = 0;
2497                 break;
2498         }
2499         return ret;
2500 }
2501 EXPORT_SYMBOL(rdma_notify);
2502
2503 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2504                 u8 private_data_len)
2505 {
2506         struct rdma_id_private *id_priv;
2507         int ret;
2508
2509         id_priv = container_of(id, struct rdma_id_private, id);
2510         if (!cma_has_cm_dev(id_priv))
2511                 return -EINVAL;
2512
2513         switch (rdma_node_get_transport(id->device->node_type)) {
2514         case RDMA_TRANSPORT_IB:
2515                 if (cma_is_ud_ps(id->ps))
2516                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2517                                                 private_data, private_data_len);
2518                 else
2519                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2520                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2521                                              0, private_data, private_data_len);
2522                 break;
2523         case RDMA_TRANSPORT_IWARP:
2524                 ret = iw_cm_reject(id_priv->cm_id.iw,
2525                                    private_data, private_data_len);
2526                 break;
2527         default:
2528                 ret = -ENOSYS;
2529                 break;
2530         }
2531         return ret;
2532 }
2533 EXPORT_SYMBOL(rdma_reject);
2534
2535 int rdma_disconnect(struct rdma_cm_id *id)
2536 {
2537         struct rdma_id_private *id_priv;
2538         int ret;
2539
2540         id_priv = container_of(id, struct rdma_id_private, id);
2541         if (!cma_has_cm_dev(id_priv))
2542                 return -EINVAL;
2543
2544         switch (rdma_node_get_transport(id->device->node_type)) {
2545         case RDMA_TRANSPORT_IB:
2546                 ret = cma_modify_qp_err(id_priv);
2547                 if (ret)
2548                         goto out;
2549                 /* Initiate or respond to a disconnect. */
2550                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2551                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2552                 break;
2553         case RDMA_TRANSPORT_IWARP:
2554                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2555                 break;
2556         default:
2557                 ret = -EINVAL;
2558                 break;
2559         }
2560 out:
2561         return ret;
2562 }
2563 EXPORT_SYMBOL(rdma_disconnect);
2564
2565 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2566 {
2567         struct rdma_id_private *id_priv;
2568         struct cma_multicast *mc = multicast->context;
2569         struct rdma_cm_event event;
2570         int ret;
2571
2572         id_priv = mc->id_priv;
2573         if (cma_disable_remove(id_priv, CMA_ADDR_BOUND) &&
2574             cma_disable_remove(id_priv, CMA_ADDR_RESOLVED))
2575                 return 0;
2576
2577         mutex_lock(&id_priv->qp_mutex);
2578         if (!status && id_priv->id.qp)
2579                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2580                                          multicast->rec.mlid);
2581         mutex_unlock(&id_priv->qp_mutex);
2582
2583         memset(&event, 0, sizeof event);
2584         event.status = status;
2585         event.param.ud.private_data = mc->context;
2586         if (!status) {
2587                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2588                 ib_init_ah_from_mcmember(id_priv->id.device,
2589                                          id_priv->id.port_num, &multicast->rec,
2590                                          &event.param.ud.ah_attr);
2591                 event.param.ud.qp_num = 0xFFFFFF;
2592                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2593         } else
2594                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2595
2596         ret = id_priv->id.event_handler(&id_priv->id, &event);
2597         if (ret) {
2598                 cma_exch(id_priv, CMA_DESTROYING);
2599                 cma_enable_remove(id_priv);
2600                 rdma_destroy_id(&id_priv->id);
2601                 return 0;
2602         }
2603
2604         cma_enable_remove(id_priv);
2605         return 0;
2606 }
2607
2608 static void cma_set_mgid(struct rdma_id_private *id_priv,
2609                          struct sockaddr *addr, union ib_gid *mgid)
2610 {
2611         unsigned char mc_map[MAX_ADDR_LEN];
2612         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2613         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2614         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2615
2616         if (cma_any_addr(addr)) {
2617                 memset(mgid, 0, sizeof *mgid);
2618         } else if ((addr->sa_family == AF_INET6) &&
2619                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2620                                                                  0xFF10A01B)) {
2621                 /* IPv6 address is an SA assigned MGID. */
2622                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2623         } else {
2624                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
2625                 if (id_priv->id.ps == RDMA_PS_UDP)
2626                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2627                 *mgid = *(union ib_gid *) (mc_map + 4);
2628         }
2629 }
2630
2631 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2632                                  struct cma_multicast *mc)
2633 {
2634         struct ib_sa_mcmember_rec rec;
2635         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2636         ib_sa_comp_mask comp_mask;
2637         int ret;
2638
2639         ib_addr_get_mgid(dev_addr, &rec.mgid);
2640         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2641                                      &rec.mgid, &rec);
2642         if (ret)
2643                 return ret;
2644
2645         cma_set_mgid(id_priv, &mc->addr, &rec.mgid);
2646         if (id_priv->id.ps == RDMA_PS_UDP)
2647                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2648         ib_addr_get_sgid(dev_addr, &rec.port_gid);
2649         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2650         rec.join_state = 1;
2651
2652         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2653                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2654                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2655                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
2656                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2657
2658         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2659                                                 id_priv->id.port_num, &rec,
2660                                                 comp_mask, GFP_KERNEL,
2661                                                 cma_ib_mc_handler, mc);
2662         if (IS_ERR(mc->multicast.ib))
2663                 return PTR_ERR(mc->multicast.ib);
2664
2665         return 0;
2666 }
2667
2668 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2669                         void *context)
2670 {
2671         struct rdma_id_private *id_priv;
2672         struct cma_multicast *mc;
2673         int ret;
2674
2675         id_priv = container_of(id, struct rdma_id_private, id);
2676         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2677             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2678                 return -EINVAL;
2679
2680         mc = kmalloc(sizeof *mc, GFP_KERNEL);
2681         if (!mc)
2682                 return -ENOMEM;
2683
2684         memcpy(&mc->addr, addr, ip_addr_size(addr));
2685         mc->context = context;
2686         mc->id_priv = id_priv;
2687
2688         spin_lock(&id_priv->lock);
2689         list_add(&mc->list, &id_priv->mc_list);
2690         spin_unlock(&id_priv->lock);
2691
2692         switch (rdma_node_get_transport(id->device->node_type)) {
2693         case RDMA_TRANSPORT_IB:
2694                 ret = cma_join_ib_multicast(id_priv, mc);
2695                 break;
2696         default:
2697                 ret = -ENOSYS;
2698                 break;
2699         }
2700
2701         if (ret) {
2702                 spin_lock_irq(&id_priv->lock);
2703                 list_del(&mc->list);
2704                 spin_unlock_irq(&id_priv->lock);
2705                 kfree(mc);
2706         }
2707         return ret;
2708 }
2709 EXPORT_SYMBOL(rdma_join_multicast);
2710
2711 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2712 {
2713         struct rdma_id_private *id_priv;
2714         struct cma_multicast *mc;
2715
2716         id_priv = container_of(id, struct rdma_id_private, id);
2717         spin_lock_irq(&id_priv->lock);
2718         list_for_each_entry(mc, &id_priv->mc_list, list) {
2719                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2720                         list_del(&mc->list);
2721                         spin_unlock_irq(&id_priv->lock);
2722
2723                         if (id->qp)
2724                                 ib_detach_mcast(id->qp,
2725                                                 &mc->multicast.ib->rec.mgid,
2726                                                 mc->multicast.ib->rec.mlid);
2727                         ib_sa_free_multicast(mc->multicast.ib);
2728                         kfree(mc);
2729                         return;
2730                 }
2731         }
2732         spin_unlock_irq(&id_priv->lock);
2733 }
2734 EXPORT_SYMBOL(rdma_leave_multicast);
2735
2736 static void cma_add_one(struct ib_device *device)
2737 {
2738         struct cma_device *cma_dev;
2739         struct rdma_id_private *id_priv;
2740
2741         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2742         if (!cma_dev)
2743                 return;
2744
2745         cma_dev->device = device;
2746
2747         init_completion(&cma_dev->comp);
2748         atomic_set(&cma_dev->refcount, 1);
2749         INIT_LIST_HEAD(&cma_dev->id_list);
2750         ib_set_client_data(device, &cma_client, cma_dev);
2751
2752         mutex_lock(&lock);
2753         list_add_tail(&cma_dev->list, &dev_list);
2754         list_for_each_entry(id_priv, &listen_any_list, list)
2755                 cma_listen_on_dev(id_priv, cma_dev);
2756         mutex_unlock(&lock);
2757 }
2758
2759 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2760 {
2761         struct rdma_cm_event event;
2762         enum cma_state state;
2763
2764         /* Record that we want to remove the device */
2765         state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2766         if (state == CMA_DESTROYING)
2767                 return 0;
2768
2769         cma_cancel_operation(id_priv, state);
2770         wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
2771
2772         /* Check for destruction from another callback. */
2773         if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2774                 return 0;
2775
2776         memset(&event, 0, sizeof event);
2777         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2778         return id_priv->id.event_handler(&id_priv->id, &event);
2779 }
2780
2781 static void cma_process_remove(struct cma_device *cma_dev)
2782 {
2783         struct rdma_id_private *id_priv;
2784         int ret;
2785
2786         mutex_lock(&lock);
2787         while (!list_empty(&cma_dev->id_list)) {
2788                 id_priv = list_entry(cma_dev->id_list.next,
2789                                      struct rdma_id_private, list);
2790
2791                 list_del(&id_priv->listen_list);
2792                 list_del_init(&id_priv->list);
2793                 atomic_inc(&id_priv->refcount);
2794                 mutex_unlock(&lock);
2795
2796                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
2797                 cma_deref_id(id_priv);
2798                 if (ret)
2799                         rdma_destroy_id(&id_priv->id);
2800
2801                 mutex_lock(&lock);
2802         }
2803         mutex_unlock(&lock);
2804
2805         cma_deref_dev(cma_dev);
2806         wait_for_completion(&cma_dev->comp);
2807 }
2808
2809 static void cma_remove_one(struct ib_device *device)
2810 {
2811         struct cma_device *cma_dev;
2812
2813         cma_dev = ib_get_client_data(device, &cma_client);
2814         if (!cma_dev)
2815                 return;
2816
2817         mutex_lock(&lock);
2818         list_del(&cma_dev->list);
2819         mutex_unlock(&lock);
2820
2821         cma_process_remove(cma_dev);
2822         kfree(cma_dev);
2823 }
2824
2825 static int cma_init(void)
2826 {
2827         int ret, low, high, remaining;
2828
2829         get_random_bytes(&next_port, sizeof next_port);
2830         inet_get_local_port_range(&low, &high);
2831         remaining = (high - low) + 1;
2832         next_port = ((unsigned int) next_port % remaining) + low;
2833
2834         cma_wq = create_singlethread_workqueue("rdma_cm");
2835         if (!cma_wq)
2836                 return -ENOMEM;
2837
2838         ib_sa_register_client(&sa_client);
2839         rdma_addr_register_client(&addr_client);
2840
2841         ret = ib_register_client(&cma_client);
2842         if (ret)
2843                 goto err;
2844         return 0;
2845
2846 err:
2847         rdma_addr_unregister_client(&addr_client);
2848         ib_sa_unregister_client(&sa_client);
2849         destroy_workqueue(cma_wq);
2850         return ret;
2851 }
2852
2853 static void cma_cleanup(void)
2854 {
2855         ib_unregister_client(&cma_client);
2856         rdma_addr_unregister_client(&addr_client);
2857         ib_sa_unregister_client(&sa_client);
2858         destroy_workqueue(cma_wq);
2859         idr_destroy(&sdp_ps);
2860         idr_destroy(&tcp_ps);
2861         idr_destroy(&udp_ps);
2862         idr_destroy(&ipoib_ps);
2863 }
2864
2865 module_init(cma_init);
2866 module_exit(cma_cleanup);