Merge HEAD from master.kernel.org:/home/rmk/linux-2.6-arm.git
[linux-2.6.git] / drivers / infiniband / core / ucm.c
1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *      copyright notice, this list of conditions and the following
16  *      disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *      copyright notice, this list of conditions and the following
20  *      disclaimer in the documentation and/or other materials
21  *      provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  * $Id: ucm.c 2594 2005-06-13 19:46:02Z libor $
33  */
34 #include <linux/init.h>
35 #include <linux/fs.h>
36 #include <linux/module.h>
37 #include <linux/device.h>
38 #include <linux/err.h>
39 #include <linux/poll.h>
40 #include <linux/file.h>
41 #include <linux/mount.h>
42 #include <linux/cdev.h>
43
44 #include <asm/uaccess.h>
45
46 #include "ucm.h"
47
48 MODULE_AUTHOR("Libor Michalek");
49 MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");
50 MODULE_LICENSE("Dual BSD/GPL");
51
52 static int ucm_debug_level;
53
54 module_param_named(debug_level, ucm_debug_level, int, 0644);
55 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
56
57 enum {
58         IB_UCM_MAJOR = 231,
59         IB_UCM_MINOR = 255
60 };
61
62 #define IB_UCM_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_MINOR)
63
64 #define PFX "UCM: "
65
66 #define ucm_dbg(format, arg...)                 \
67         do {                                    \
68                 if (ucm_debug_level > 0)        \
69                         printk(KERN_DEBUG PFX format, ## arg); \
70         } while (0)
71
72 static struct semaphore ctx_id_mutex;
73 static struct idr       ctx_id_table;
74 static int              ctx_id_rover = 0;
75
76 static struct ib_ucm_context *ib_ucm_ctx_get(int id)
77 {
78         struct ib_ucm_context *ctx;
79
80         down(&ctx_id_mutex);
81         ctx = idr_find(&ctx_id_table, id);
82         if (ctx)
83                 ctx->ref++;
84         up(&ctx_id_mutex);
85
86         return ctx;
87 }
88
89 static void ib_ucm_ctx_put(struct ib_ucm_context *ctx)
90 {
91         struct ib_ucm_event *uevent;
92
93         down(&ctx_id_mutex);
94
95         ctx->ref--;
96         if (!ctx->ref)
97                 idr_remove(&ctx_id_table, ctx->id);
98
99         up(&ctx_id_mutex);
100
101         if (ctx->ref)
102                 return;
103
104         down(&ctx->file->mutex);
105
106         list_del(&ctx->file_list);
107         while (!list_empty(&ctx->events)) {
108
109                 uevent = list_entry(ctx->events.next,
110                                     struct ib_ucm_event, ctx_list);
111                 list_del(&uevent->file_list);
112                 list_del(&uevent->ctx_list);
113
114                 /* clear incoming connections. */
115                 if (uevent->cm_id)
116                         ib_destroy_cm_id(uevent->cm_id);
117
118                 kfree(uevent);
119         }
120
121         up(&ctx->file->mutex);
122
123         ucm_dbg("Destroyed CM ID <%d>\n", ctx->id);
124
125         ib_destroy_cm_id(ctx->cm_id);
126         kfree(ctx);
127 }
128
129 static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file)
130 {
131         struct ib_ucm_context *ctx;
132         int result;
133
134         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
135         if (!ctx)
136                 return NULL;
137
138         ctx->ref  = 1; /* user reference */
139         ctx->file = file;
140
141         INIT_LIST_HEAD(&ctx->events);
142         init_MUTEX(&ctx->mutex);
143
144         list_add_tail(&ctx->file_list, &file->ctxs);
145
146         ctx_id_rover = (ctx_id_rover + 1) & INT_MAX;
147 retry:
148         result = idr_pre_get(&ctx_id_table, GFP_KERNEL);
149         if (!result)
150                 goto error;
151
152         down(&ctx_id_mutex);
153         result = idr_get_new_above(&ctx_id_table, ctx, ctx_id_rover, &ctx->id);
154         up(&ctx_id_mutex);
155
156         if (result == -EAGAIN)
157                 goto retry;
158         if (result)
159                 goto error;
160
161         ucm_dbg("Allocated CM ID <%d>\n", ctx->id);
162
163         return ctx;
164 error:
165         list_del(&ctx->file_list);
166         kfree(ctx);
167
168         return NULL;
169 }
170 /*
171  * Event portion of the API, handle CM events
172  * and allow event polling.
173  */
174 static void ib_ucm_event_path_get(struct ib_ucm_path_rec *upath,
175                                   struct ib_sa_path_rec  *kpath)
176 {
177         if (!kpath || !upath)
178                 return;
179
180         memcpy(upath->dgid, kpath->dgid.raw, sizeof(union ib_gid));
181         memcpy(upath->sgid, kpath->sgid.raw, sizeof(union ib_gid));
182
183         upath->dlid             = kpath->dlid;
184         upath->slid             = kpath->slid;
185         upath->raw_traffic      = kpath->raw_traffic;
186         upath->flow_label       = kpath->flow_label;
187         upath->hop_limit        = kpath->hop_limit;
188         upath->traffic_class    = kpath->traffic_class;
189         upath->reversible       = kpath->reversible;
190         upath->numb_path        = kpath->numb_path;
191         upath->pkey             = kpath->pkey;
192         upath->sl               = kpath->sl;
193         upath->mtu_selector     = kpath->mtu_selector;
194         upath->mtu              = kpath->mtu;
195         upath->rate_selector    = kpath->rate_selector;
196         upath->rate             = kpath->rate;
197         upath->packet_life_time = kpath->packet_life_time;
198         upath->preference       = kpath->preference;
199
200         upath->packet_life_time_selector =
201                 kpath->packet_life_time_selector;
202 }
203
204 static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq,
205                                  struct ib_cm_req_event_param *kreq)
206 {
207         ureq->listen_id = (long)kreq->listen_id->context;
208
209         ureq->remote_ca_guid             = kreq->remote_ca_guid;
210         ureq->remote_qkey                = kreq->remote_qkey;
211         ureq->remote_qpn                 = kreq->remote_qpn;
212         ureq->qp_type                    = kreq->qp_type;
213         ureq->starting_psn               = kreq->starting_psn;
214         ureq->responder_resources        = kreq->responder_resources;
215         ureq->initiator_depth            = kreq->initiator_depth;
216         ureq->local_cm_response_timeout  = kreq->local_cm_response_timeout;
217         ureq->flow_control               = kreq->flow_control;
218         ureq->remote_cm_response_timeout = kreq->remote_cm_response_timeout;
219         ureq->retry_count                = kreq->retry_count;
220         ureq->rnr_retry_count            = kreq->rnr_retry_count;
221         ureq->srq                        = kreq->srq;
222
223         ib_ucm_event_path_get(&ureq->primary_path, kreq->primary_path);
224         ib_ucm_event_path_get(&ureq->alternate_path, kreq->alternate_path);
225 }
226
227 static void ib_ucm_event_rep_get(struct ib_ucm_rep_event_resp *urep,
228                                  struct ib_cm_rep_event_param *krep)
229 {
230         urep->remote_ca_guid      = krep->remote_ca_guid;
231         urep->remote_qkey         = krep->remote_qkey;
232         urep->remote_qpn          = krep->remote_qpn;
233         urep->starting_psn        = krep->starting_psn;
234         urep->responder_resources = krep->responder_resources;
235         urep->initiator_depth     = krep->initiator_depth;
236         urep->target_ack_delay    = krep->target_ack_delay;
237         urep->failover_accepted   = krep->failover_accepted;
238         urep->flow_control        = krep->flow_control;
239         urep->rnr_retry_count     = krep->rnr_retry_count;
240         urep->srq                 = krep->srq;
241 }
242
243 static void ib_ucm_event_rej_get(struct ib_ucm_rej_event_resp *urej,
244                                  struct ib_cm_rej_event_param *krej)
245 {
246         urej->reason = krej->reason;
247 }
248
249 static void ib_ucm_event_mra_get(struct ib_ucm_mra_event_resp *umra,
250                                  struct ib_cm_mra_event_param *kmra)
251 {
252         umra->timeout = kmra->service_timeout;
253 }
254
255 static void ib_ucm_event_lap_get(struct ib_ucm_lap_event_resp *ulap,
256                                  struct ib_cm_lap_event_param *klap)
257 {
258         ib_ucm_event_path_get(&ulap->path, klap->alternate_path);
259 }
260
261 static void ib_ucm_event_apr_get(struct ib_ucm_apr_event_resp *uapr,
262                                  struct ib_cm_apr_event_param *kapr)
263 {
264         uapr->status = kapr->ap_status;
265 }
266
267 static void ib_ucm_event_sidr_req_get(struct ib_ucm_sidr_req_event_resp *ureq,
268                                       struct ib_cm_sidr_req_event_param *kreq)
269 {
270         ureq->listen_id = (long)kreq->listen_id->context;
271         ureq->pkey      = kreq->pkey;
272 }
273
274 static void ib_ucm_event_sidr_rep_get(struct ib_ucm_sidr_rep_event_resp *urep,
275                                       struct ib_cm_sidr_rep_event_param *krep)
276 {
277         urep->status = krep->status;
278         urep->qkey   = krep->qkey;
279         urep->qpn    = krep->qpn;
280 };
281
282 static int ib_ucm_event_process(struct ib_cm_event *evt,
283                                 struct ib_ucm_event *uvt)
284 {
285         void *info = NULL;
286         int result;
287
288         switch (evt->event) {
289         case IB_CM_REQ_RECEIVED:
290                 ib_ucm_event_req_get(&uvt->resp.u.req_resp,
291                                      &evt->param.req_rcvd);
292                 uvt->data_len      = IB_CM_REQ_PRIVATE_DATA_SIZE;
293                 uvt->resp.present |= (evt->param.req_rcvd.primary_path ?
294                                       IB_UCM_PRES_PRIMARY : 0);
295                 uvt->resp.present |= (evt->param.req_rcvd.alternate_path ?
296                                       IB_UCM_PRES_ALTERNATE : 0);
297                 break;
298         case IB_CM_REP_RECEIVED:
299                 ib_ucm_event_rep_get(&uvt->resp.u.rep_resp,
300                                      &evt->param.rep_rcvd);
301                 uvt->data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
302
303                 break;
304         case IB_CM_RTU_RECEIVED:
305                 uvt->data_len = IB_CM_RTU_PRIVATE_DATA_SIZE;
306                 uvt->resp.u.send_status = evt->param.send_status;
307
308                 break;
309         case IB_CM_DREQ_RECEIVED:
310                 uvt->data_len = IB_CM_DREQ_PRIVATE_DATA_SIZE;
311                 uvt->resp.u.send_status = evt->param.send_status;
312
313                 break;
314         case IB_CM_DREP_RECEIVED:
315                 uvt->data_len = IB_CM_DREP_PRIVATE_DATA_SIZE;
316                 uvt->resp.u.send_status = evt->param.send_status;
317
318                 break;
319         case IB_CM_MRA_RECEIVED:
320                 ib_ucm_event_mra_get(&uvt->resp.u.mra_resp,
321                                      &evt->param.mra_rcvd);
322                 uvt->data_len = IB_CM_MRA_PRIVATE_DATA_SIZE;
323
324                 break;
325         case IB_CM_REJ_RECEIVED:
326                 ib_ucm_event_rej_get(&uvt->resp.u.rej_resp,
327                                      &evt->param.rej_rcvd);
328                 uvt->data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
329                 uvt->info_len = evt->param.rej_rcvd.ari_length;
330                 info          = evt->param.rej_rcvd.ari;
331
332                 break;
333         case IB_CM_LAP_RECEIVED:
334                 ib_ucm_event_lap_get(&uvt->resp.u.lap_resp,
335                                      &evt->param.lap_rcvd);
336                 uvt->data_len = IB_CM_LAP_PRIVATE_DATA_SIZE;
337                 uvt->resp.present |= (evt->param.lap_rcvd.alternate_path ?
338                                       IB_UCM_PRES_ALTERNATE : 0);
339                 break;
340         case IB_CM_APR_RECEIVED:
341                 ib_ucm_event_apr_get(&uvt->resp.u.apr_resp,
342                                      &evt->param.apr_rcvd);
343                 uvt->data_len = IB_CM_APR_PRIVATE_DATA_SIZE;
344                 uvt->info_len = evt->param.apr_rcvd.info_len;
345                 info          = evt->param.apr_rcvd.apr_info;
346
347                 break;
348         case IB_CM_SIDR_REQ_RECEIVED:
349                 ib_ucm_event_sidr_req_get(&uvt->resp.u.sidr_req_resp,
350                                           &evt->param.sidr_req_rcvd);
351                 uvt->data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE;
352
353                 break;
354         case IB_CM_SIDR_REP_RECEIVED:
355                 ib_ucm_event_sidr_rep_get(&uvt->resp.u.sidr_rep_resp,
356                                           &evt->param.sidr_rep_rcvd);
357                 uvt->data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
358                 uvt->info_len = evt->param.sidr_rep_rcvd.info_len;
359                 info          = evt->param.sidr_rep_rcvd.info;
360
361                 break;
362         default:
363                 uvt->resp.u.send_status = evt->param.send_status;
364
365                 break;
366         }
367
368         if (uvt->data_len && evt->private_data) {
369
370                 uvt->data = kmalloc(uvt->data_len, GFP_KERNEL);
371                 if (!uvt->data) {
372                         result = -ENOMEM;
373                         goto error;
374                 }
375
376                 memcpy(uvt->data, evt->private_data, uvt->data_len);
377                 uvt->resp.present |= IB_UCM_PRES_DATA;
378         }
379
380         if (uvt->info_len && info) {
381
382                 uvt->info = kmalloc(uvt->info_len, GFP_KERNEL);
383                 if (!uvt->info) {
384                         result = -ENOMEM;
385                         goto error;
386                 }
387
388                 memcpy(uvt->info, info, uvt->info_len);
389                 uvt->resp.present |= IB_UCM_PRES_INFO;
390         }
391
392         return 0;
393 error:
394         kfree(uvt->info);
395         kfree(uvt->data);
396         return result;
397 }
398
399 static int ib_ucm_event_handler(struct ib_cm_id *cm_id,
400                                 struct ib_cm_event *event)
401 {
402         struct ib_ucm_event *uevent;
403         struct ib_ucm_context *ctx;
404         int result = 0;
405         int id;
406         /*
407          * lookup correct context based on event type.
408          */
409         switch (event->event) {
410         case IB_CM_REQ_RECEIVED:
411                 id = (long)event->param.req_rcvd.listen_id->context;
412                 break;
413         case IB_CM_SIDR_REQ_RECEIVED:
414                 id = (long)event->param.sidr_req_rcvd.listen_id->context;
415                 break;
416         default:
417                 id = (long)cm_id->context;
418                 break;
419         }
420
421         ucm_dbg("Event. CM ID <%d> event <%d>\n", id, event->event);
422
423         ctx = ib_ucm_ctx_get(id);
424         if (!ctx)
425                 return -ENOENT;
426
427         if (event->event == IB_CM_REQ_RECEIVED ||
428             event->event == IB_CM_SIDR_REQ_RECEIVED)
429                 id = IB_UCM_CM_ID_INVALID;
430
431         uevent = kmalloc(sizeof(*uevent), GFP_KERNEL);
432         if (!uevent) {
433                 result = -ENOMEM;
434                 goto done;
435         }
436
437         memset(uevent, 0, sizeof(*uevent));
438
439         uevent->resp.id    = id;
440         uevent->resp.event = event->event;
441
442         result = ib_ucm_event_process(event, uevent);
443         if (result)
444                 goto done;
445
446         uevent->ctx   = ctx;
447         uevent->cm_id = ((event->event == IB_CM_REQ_RECEIVED ||
448                           event->event == IB_CM_SIDR_REQ_RECEIVED ) ?
449                          cm_id : NULL);
450
451         down(&ctx->file->mutex);
452
453         list_add_tail(&uevent->file_list, &ctx->file->events);
454         list_add_tail(&uevent->ctx_list, &ctx->events);
455
456         wake_up_interruptible(&ctx->file->poll_wait);
457
458         up(&ctx->file->mutex);
459 done:
460         ctx->error = result;
461         ib_ucm_ctx_put(ctx); /* func reference */
462         return result;
463 }
464
465 static ssize_t ib_ucm_event(struct ib_ucm_file *file,
466                             const char __user *inbuf,
467                             int in_len, int out_len)
468 {
469         struct ib_ucm_context *ctx;
470         struct ib_ucm_event_get cmd;
471         struct ib_ucm_event *uevent = NULL;
472         int result = 0;
473         DEFINE_WAIT(wait);
474
475         if (out_len < sizeof(struct ib_ucm_event_resp))
476                 return -ENOSPC;
477
478         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
479                 return -EFAULT;
480         /*
481          * wait
482          */
483         down(&file->mutex);
484
485         while (list_empty(&file->events)) {
486
487                 if (file->filp->f_flags & O_NONBLOCK) {
488                         result = -EAGAIN;
489                         break;
490                 }
491
492                 if (signal_pending(current)) {
493                         result = -ERESTARTSYS;
494                         break;
495                 }
496
497                 prepare_to_wait(&file->poll_wait, &wait, TASK_INTERRUPTIBLE);
498
499                 up(&file->mutex);
500                 schedule();
501                 down(&file->mutex);
502
503                 finish_wait(&file->poll_wait, &wait);
504         }
505
506         if (result)
507                 goto done;
508
509         uevent = list_entry(file->events.next, struct ib_ucm_event, file_list);
510
511         if (!uevent->cm_id)
512                 goto user;
513
514         ctx = ib_ucm_ctx_alloc(file);
515         if (!ctx) {
516                 result = -ENOMEM;
517                 goto done;
518         }
519
520         ctx->cm_id             = uevent->cm_id;
521         ctx->cm_id->cm_handler = ib_ucm_event_handler;
522         ctx->cm_id->context    = (void *)(unsigned long)ctx->id;
523
524         uevent->resp.id = ctx->id;
525
526 user:
527         if (copy_to_user((void __user *)(unsigned long)cmd.response,
528                          &uevent->resp, sizeof(uevent->resp))) {
529                 result = -EFAULT;
530                 goto done;
531         }
532
533         if (uevent->data) {
534
535                 if (cmd.data_len < uevent->data_len) {
536                         result = -ENOMEM;
537                         goto done;
538                 }
539
540                 if (copy_to_user((void __user *)(unsigned long)cmd.data,
541                                  uevent->data, uevent->data_len)) {
542                         result = -EFAULT;
543                         goto done;
544                 }
545         }
546
547         if (uevent->info) {
548
549                 if (cmd.info_len < uevent->info_len) {
550                         result = -ENOMEM;
551                         goto done;
552                 }
553
554                 if (copy_to_user((void __user *)(unsigned long)cmd.info,
555                                  uevent->info, uevent->info_len)) {
556                         result = -EFAULT;
557                         goto done;
558                 }
559         }
560
561         list_del(&uevent->file_list);
562         list_del(&uevent->ctx_list);
563
564         kfree(uevent->data);
565         kfree(uevent->info);
566         kfree(uevent);
567 done:
568         up(&file->mutex);
569         return result;
570 }
571
572
573 static ssize_t ib_ucm_create_id(struct ib_ucm_file *file,
574                                 const char __user *inbuf,
575                                 int in_len, int out_len)
576 {
577         struct ib_ucm_create_id cmd;
578         struct ib_ucm_create_id_resp resp;
579         struct ib_ucm_context *ctx;
580         int result;
581
582         if (out_len < sizeof(resp))
583                 return -ENOSPC;
584
585         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
586                 return -EFAULT;
587
588         ctx = ib_ucm_ctx_alloc(file);
589         if (!ctx)
590                 return -ENOMEM;
591
592         ctx->cm_id = ib_create_cm_id(ib_ucm_event_handler,
593                                      (void *)(unsigned long)ctx->id);
594         if (!ctx->cm_id) {
595                 result = -ENOMEM;
596                 goto err_cm;
597         }
598
599         resp.id = ctx->id;
600         if (copy_to_user((void __user *)(unsigned long)cmd.response,
601                          &resp, sizeof(resp))) {
602                 result = -EFAULT;
603                 goto err_ret;
604         }
605
606         return 0;
607 err_ret:
608         ib_destroy_cm_id(ctx->cm_id);
609 err_cm:
610         ib_ucm_ctx_put(ctx); /* user reference */
611
612         return result;
613 }
614
615 static ssize_t ib_ucm_destroy_id(struct ib_ucm_file *file,
616                                  const char __user *inbuf,
617                                  int in_len, int out_len)
618 {
619         struct ib_ucm_destroy_id cmd;
620         struct ib_ucm_context *ctx;
621
622         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
623                 return -EFAULT;
624
625         ctx = ib_ucm_ctx_get(cmd.id);
626         if (!ctx)
627                 return -ENOENT;
628
629         ib_ucm_ctx_put(ctx); /* user reference */
630         ib_ucm_ctx_put(ctx); /* func reference */
631
632         return 0;
633 }
634
635 static ssize_t ib_ucm_attr_id(struct ib_ucm_file *file,
636                               const char __user *inbuf,
637                               int in_len, int out_len)
638 {
639         struct ib_ucm_attr_id_resp resp;
640         struct ib_ucm_attr_id cmd;
641         struct ib_ucm_context *ctx;
642         int result = 0;
643
644         if (out_len < sizeof(resp))
645                 return -ENOSPC;
646
647         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
648                 return -EFAULT;
649
650         ctx = ib_ucm_ctx_get(cmd.id);
651         if (!ctx)
652                 return -ENOENT;
653
654         down(&ctx->file->mutex);
655         if (ctx->file != file) {
656                 result = -EINVAL;
657                 goto done;
658         }
659
660         resp.service_id   = ctx->cm_id->service_id;
661         resp.service_mask = ctx->cm_id->service_mask;
662         resp.local_id     = ctx->cm_id->local_id;
663         resp.remote_id    = ctx->cm_id->remote_id;
664
665         if (copy_to_user((void __user *)(unsigned long)cmd.response,
666                          &resp, sizeof(resp)))
667                 result = -EFAULT;
668
669 done:
670         up(&ctx->file->mutex);
671         ib_ucm_ctx_put(ctx); /* func reference */
672         return result;
673 }
674
675 static ssize_t ib_ucm_listen(struct ib_ucm_file *file,
676                              const char __user *inbuf,
677                              int in_len, int out_len)
678 {
679         struct ib_ucm_listen cmd;
680         struct ib_ucm_context *ctx;
681         int result;
682
683         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
684                 return -EFAULT;
685
686         ctx = ib_ucm_ctx_get(cmd.id);
687         if (!ctx)
688                 return -ENOENT;
689
690         down(&ctx->file->mutex);
691         if (ctx->file != file)
692                 result = -EINVAL;
693         else
694                 result = ib_cm_listen(ctx->cm_id, cmd.service_id,
695                                       cmd.service_mask);
696
697         up(&ctx->file->mutex);
698         ib_ucm_ctx_put(ctx); /* func reference */
699         return result;
700 }
701
702 static ssize_t ib_ucm_establish(struct ib_ucm_file *file,
703                                 const char __user *inbuf,
704                                 int in_len, int out_len)
705 {
706         struct ib_ucm_establish cmd;
707         struct ib_ucm_context *ctx;
708         int result;
709
710         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
711                 return -EFAULT;
712
713         ctx = ib_ucm_ctx_get(cmd.id);
714         if (!ctx)
715                 return -ENOENT;
716
717         down(&ctx->file->mutex);
718         if (ctx->file != file)
719                 result = -EINVAL;
720         else
721                 result = ib_cm_establish(ctx->cm_id);
722
723         up(&ctx->file->mutex);
724         ib_ucm_ctx_put(ctx); /* func reference */
725         return result;
726 }
727
728 static int ib_ucm_alloc_data(const void **dest, u64 src, u32 len)
729 {
730         void *data;
731
732         *dest = NULL;
733
734         if (!len)
735                 return 0;
736
737         data = kmalloc(len, GFP_KERNEL);
738         if (!data)
739                 return -ENOMEM;
740
741         if (copy_from_user(data, (void __user *)(unsigned long)src, len)) {
742                 kfree(data);
743                 return -EFAULT;
744         }
745
746         *dest = data;
747         return 0;
748 }
749
750 static int ib_ucm_path_get(struct ib_sa_path_rec **path, u64 src)
751 {
752         struct ib_ucm_path_rec ucm_path;
753         struct ib_sa_path_rec  *sa_path;
754
755         *path = NULL;
756
757         if (!src)
758                 return 0;
759
760         sa_path = kmalloc(sizeof(*sa_path), GFP_KERNEL);
761         if (!sa_path)
762                 return -ENOMEM;
763
764         if (copy_from_user(&ucm_path, (void __user *)(unsigned long)src,
765                            sizeof(ucm_path))) {
766
767                 kfree(sa_path);
768                 return -EFAULT;
769         }
770
771         memcpy(sa_path->dgid.raw, ucm_path.dgid, sizeof(union ib_gid));
772         memcpy(sa_path->sgid.raw, ucm_path.sgid, sizeof(union ib_gid));
773
774         sa_path->dlid             = ucm_path.dlid;
775         sa_path->slid             = ucm_path.slid;
776         sa_path->raw_traffic      = ucm_path.raw_traffic;
777         sa_path->flow_label       = ucm_path.flow_label;
778         sa_path->hop_limit        = ucm_path.hop_limit;
779         sa_path->traffic_class    = ucm_path.traffic_class;
780         sa_path->reversible       = ucm_path.reversible;
781         sa_path->numb_path        = ucm_path.numb_path;
782         sa_path->pkey             = ucm_path.pkey;
783         sa_path->sl               = ucm_path.sl;
784         sa_path->mtu_selector     = ucm_path.mtu_selector;
785         sa_path->mtu              = ucm_path.mtu;
786         sa_path->rate_selector    = ucm_path.rate_selector;
787         sa_path->rate             = ucm_path.rate;
788         sa_path->packet_life_time = ucm_path.packet_life_time;
789         sa_path->preference       = ucm_path.preference;
790
791         sa_path->packet_life_time_selector =
792                 ucm_path.packet_life_time_selector;
793
794         *path = sa_path;
795         return 0;
796 }
797
798 static ssize_t ib_ucm_send_req(struct ib_ucm_file *file,
799                                const char __user *inbuf,
800                                int in_len, int out_len)
801 {
802         struct ib_cm_req_param param;
803         struct ib_ucm_context *ctx;
804         struct ib_ucm_req cmd;
805         int result;
806
807         param.private_data   = NULL;
808         param.primary_path   = NULL;
809         param.alternate_path = NULL;
810
811         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
812                 return -EFAULT;
813
814         result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
815         if (result)
816                 goto done;
817
818         result = ib_ucm_path_get(&param.primary_path, cmd.primary_path);
819         if (result)
820                 goto done;
821
822         result = ib_ucm_path_get(&param.alternate_path, cmd.alternate_path);
823         if (result)
824                 goto done;
825
826         param.private_data_len           = cmd.len;
827         param.service_id                 = cmd.sid;
828         param.qp_num                     = cmd.qpn;
829         param.qp_type                    = cmd.qp_type;
830         param.starting_psn               = cmd.psn;
831         param.peer_to_peer               = cmd.peer_to_peer;
832         param.responder_resources        = cmd.responder_resources;
833         param.initiator_depth            = cmd.initiator_depth;
834         param.remote_cm_response_timeout = cmd.remote_cm_response_timeout;
835         param.flow_control               = cmd.flow_control;
836         param.local_cm_response_timeout  = cmd.local_cm_response_timeout;
837         param.retry_count                = cmd.retry_count;
838         param.rnr_retry_count            = cmd.rnr_retry_count;
839         param.max_cm_retries             = cmd.max_cm_retries;
840         param.srq                        = cmd.srq;
841
842         ctx = ib_ucm_ctx_get(cmd.id);
843         if (!ctx) {
844                 result = -ENOENT;
845                 goto done;
846         }
847
848         down(&ctx->file->mutex);
849         if (ctx->file != file)
850                 result = -EINVAL;
851         else
852                 result = ib_send_cm_req(ctx->cm_id, &param);
853
854         up(&ctx->file->mutex);
855         ib_ucm_ctx_put(ctx); /* func reference */
856 done:
857         kfree(param.private_data);
858         kfree(param.primary_path);
859         kfree(param.alternate_path);
860
861         return result;
862 }
863
864 static ssize_t ib_ucm_send_rep(struct ib_ucm_file *file,
865                                const char __user *inbuf,
866                                int in_len, int out_len)
867 {
868         struct ib_cm_rep_param param;
869         struct ib_ucm_context *ctx;
870         struct ib_ucm_rep cmd;
871         int result;
872
873         param.private_data = NULL;
874
875         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
876                 return -EFAULT;
877
878         result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
879         if (result)
880                 return result;
881
882         param.qp_num              = cmd.qpn;
883         param.starting_psn        = cmd.psn;
884         param.private_data_len    = cmd.len;
885         param.responder_resources = cmd.responder_resources;
886         param.initiator_depth     = cmd.initiator_depth;
887         param.target_ack_delay    = cmd.target_ack_delay;
888         param.failover_accepted   = cmd.failover_accepted;
889         param.flow_control        = cmd.flow_control;
890         param.rnr_retry_count     = cmd.rnr_retry_count;
891         param.srq                 = cmd.srq;
892
893         ctx = ib_ucm_ctx_get(cmd.id);
894         if (!ctx) {
895                 result = -ENOENT;
896                 goto done;
897         }
898
899         down(&ctx->file->mutex);
900         if (ctx->file != file)
901                 result = -EINVAL;
902         else
903                 result = ib_send_cm_rep(ctx->cm_id, &param);
904
905         up(&ctx->file->mutex);
906         ib_ucm_ctx_put(ctx); /* func reference */
907 done:
908         kfree(param.private_data);
909
910         return result;
911 }
912
913 static ssize_t ib_ucm_send_private_data(struct ib_ucm_file *file,
914                                         const char __user *inbuf, int in_len,
915                                         int (*func)(struct ib_cm_id *cm_id,
916                                                     const void *private_data,
917                                                     u8 private_data_len))
918 {
919         struct ib_ucm_private_data cmd;
920         struct ib_ucm_context *ctx;
921         const void *private_data = NULL;
922         int result;
923
924         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
925                 return -EFAULT;
926
927         result = ib_ucm_alloc_data(&private_data, cmd.data, cmd.len);
928         if (result)
929                 return result;
930
931         ctx = ib_ucm_ctx_get(cmd.id);
932         if (!ctx) {
933                 result = -ENOENT;
934                 goto done;
935         }
936
937         down(&ctx->file->mutex);
938         if (ctx->file != file)
939                 result = -EINVAL;
940         else
941                 result = func(ctx->cm_id, private_data, cmd.len);
942
943         up(&ctx->file->mutex);
944         ib_ucm_ctx_put(ctx); /* func reference */
945 done:
946         kfree(private_data);
947
948         return result;
949 }
950
951 static ssize_t ib_ucm_send_rtu(struct ib_ucm_file *file,
952                                const char __user *inbuf,
953                                int in_len, int out_len)
954 {
955         return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_rtu);
956 }
957
958 static ssize_t ib_ucm_send_dreq(struct ib_ucm_file *file,
959                                 const char __user *inbuf,
960                                 int in_len, int out_len)
961 {
962         return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_dreq);
963 }
964
965 static ssize_t ib_ucm_send_drep(struct ib_ucm_file *file,
966                                 const char __user *inbuf,
967                                 int in_len, int out_len)
968 {
969         return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_drep);
970 }
971
972 static ssize_t ib_ucm_send_info(struct ib_ucm_file *file,
973                                 const char __user *inbuf, int in_len,
974                                 int (*func)(struct ib_cm_id *cm_id,
975                                             int status,
976                                             const void *info,
977                                             u8 info_len,
978                                             const void *data,
979                                             u8 data_len))
980 {
981         struct ib_ucm_context *ctx;
982         struct ib_ucm_info cmd;
983         const void *data = NULL;
984         const void *info = NULL;
985         int result;
986
987         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
988                 return -EFAULT;
989
990         result = ib_ucm_alloc_data(&data, cmd.data, cmd.data_len);
991         if (result)
992                 goto done;
993
994         result = ib_ucm_alloc_data(&info, cmd.info, cmd.info_len);
995         if (result)
996                 goto done;
997
998         ctx = ib_ucm_ctx_get(cmd.id);
999         if (!ctx) {
1000                 result = -ENOENT;
1001                 goto done;
1002         }
1003
1004         down(&ctx->file->mutex);
1005         if (ctx->file != file)
1006                 result = -EINVAL;
1007         else
1008                 result = func(ctx->cm_id, cmd.status,
1009                               info, cmd.info_len,
1010                               data, cmd.data_len);
1011
1012         up(&ctx->file->mutex);
1013         ib_ucm_ctx_put(ctx); /* func reference */
1014 done:
1015         kfree(data);
1016         kfree(info);
1017
1018         return result;
1019 }
1020
1021 static ssize_t ib_ucm_send_rej(struct ib_ucm_file *file,
1022                                const char __user *inbuf,
1023                                int in_len, int out_len)
1024 {
1025         return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_rej);
1026 }
1027
1028 static ssize_t ib_ucm_send_apr(struct ib_ucm_file *file,
1029                                const char __user *inbuf,
1030                                int in_len, int out_len)
1031 {
1032         return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_apr);
1033 }
1034
1035 static ssize_t ib_ucm_send_mra(struct ib_ucm_file *file,
1036                                const char __user *inbuf,
1037                                int in_len, int out_len)
1038 {
1039         struct ib_ucm_context *ctx;
1040         struct ib_ucm_mra cmd;
1041         const void *data = NULL;
1042         int result;
1043
1044         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
1045                 return -EFAULT;
1046
1047         result = ib_ucm_alloc_data(&data, cmd.data, cmd.len);
1048         if (result)
1049                 return result;
1050
1051         ctx = ib_ucm_ctx_get(cmd.id);
1052         if (!ctx) {
1053                 result = -ENOENT;
1054                 goto done;
1055         }
1056
1057         down(&ctx->file->mutex);
1058         if (ctx->file != file)
1059                 result = -EINVAL;
1060         else
1061                 result = ib_send_cm_mra(ctx->cm_id, cmd.timeout,
1062                                         data, cmd.len);
1063
1064         up(&ctx->file->mutex);
1065         ib_ucm_ctx_put(ctx); /* func reference */
1066 done:
1067         kfree(data);
1068
1069         return result;
1070 }
1071
1072 static ssize_t ib_ucm_send_lap(struct ib_ucm_file *file,
1073                                const char __user *inbuf,
1074                                int in_len, int out_len)
1075 {
1076         struct ib_ucm_context *ctx;
1077         struct ib_sa_path_rec *path = NULL;
1078         struct ib_ucm_lap cmd;
1079         const void *data = NULL;
1080         int result;
1081
1082         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
1083                 return -EFAULT;
1084
1085         result = ib_ucm_alloc_data(&data, cmd.data, cmd.len);
1086         if (result)
1087                 goto done;
1088
1089         result = ib_ucm_path_get(&path, cmd.path);
1090         if (result)
1091                 goto done;
1092
1093         ctx = ib_ucm_ctx_get(cmd.id);
1094         if (!ctx) {
1095                 result = -ENOENT;
1096                 goto done;
1097         }
1098
1099         down(&ctx->file->mutex);
1100         if (ctx->file != file)
1101                 result = -EINVAL;
1102         else
1103                 result = ib_send_cm_lap(ctx->cm_id, path, data, cmd.len);
1104
1105         up(&ctx->file->mutex);
1106         ib_ucm_ctx_put(ctx); /* func reference */
1107 done:
1108         kfree(data);
1109         kfree(path);
1110
1111         return result;
1112 }
1113
1114 static ssize_t ib_ucm_send_sidr_req(struct ib_ucm_file *file,
1115                                     const char __user *inbuf,
1116                                     int in_len, int out_len)
1117 {
1118         struct ib_cm_sidr_req_param param;
1119         struct ib_ucm_context *ctx;
1120         struct ib_ucm_sidr_req cmd;
1121         int result;
1122
1123         param.private_data = NULL;
1124         param.path = NULL;
1125
1126         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
1127                 return -EFAULT;
1128
1129         result = ib_ucm_alloc_data(&param.private_data, cmd.data, cmd.len);
1130         if (result)
1131                 goto done;
1132
1133         result = ib_ucm_path_get(&param.path, cmd.path);
1134         if (result)
1135                 goto done;
1136
1137         param.private_data_len = cmd.len;
1138         param.service_id       = cmd.sid;
1139         param.timeout_ms       = cmd.timeout;
1140         param.max_cm_retries   = cmd.max_cm_retries;
1141         param.pkey             = cmd.pkey;
1142
1143         ctx = ib_ucm_ctx_get(cmd.id);
1144         if (!ctx) {
1145                 result = -ENOENT;
1146                 goto done;
1147         }
1148
1149         down(&ctx->file->mutex);
1150         if (ctx->file != file)
1151                 result = -EINVAL;
1152         else
1153                 result = ib_send_cm_sidr_req(ctx->cm_id, &param);
1154
1155         up(&ctx->file->mutex);
1156         ib_ucm_ctx_put(ctx); /* func reference */
1157 done:
1158         kfree(param.private_data);
1159         kfree(param.path);
1160
1161         return result;
1162 }
1163
1164 static ssize_t ib_ucm_send_sidr_rep(struct ib_ucm_file *file,
1165                                     const char __user *inbuf,
1166                                     int in_len, int out_len)
1167 {
1168         struct ib_cm_sidr_rep_param param;
1169         struct ib_ucm_sidr_rep cmd;
1170         struct ib_ucm_context *ctx;
1171         int result;
1172
1173         param.info = NULL;
1174
1175         if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
1176                 return -EFAULT;
1177
1178         result = ib_ucm_alloc_data(&param.private_data,
1179                                    cmd.data, cmd.data_len);
1180         if (result)
1181                 goto done;
1182
1183         result = ib_ucm_alloc_data(&param.info, cmd.info, cmd.info_len);
1184         if (result)
1185                 goto done;
1186
1187         param.qp_num       = cmd.qpn;
1188         param.qkey           = cmd.qkey;
1189         param.status       = cmd.status;
1190         param.info_length      = cmd.info_len;
1191         param.private_data_len = cmd.data_len;
1192
1193         ctx = ib_ucm_ctx_get(cmd.id);
1194         if (!ctx) {
1195                 result = -ENOENT;
1196                 goto done;
1197         }
1198
1199         down(&ctx->file->mutex);
1200         if (ctx->file != file)
1201                 result = -EINVAL;
1202         else
1203                 result = ib_send_cm_sidr_rep(ctx->cm_id, &param);
1204
1205         up(&ctx->file->mutex);
1206         ib_ucm_ctx_put(ctx); /* func reference */
1207 done:
1208         kfree(param.private_data);
1209         kfree(param.info);
1210
1211         return result;
1212 }
1213
1214 static ssize_t (*ucm_cmd_table[])(struct ib_ucm_file *file,
1215                                   const char __user *inbuf,
1216                                   int in_len, int out_len) = {
1217         [IB_USER_CM_CMD_CREATE_ID]     = ib_ucm_create_id,
1218         [IB_USER_CM_CMD_DESTROY_ID]    = ib_ucm_destroy_id,
1219         [IB_USER_CM_CMD_ATTR_ID]       = ib_ucm_attr_id,
1220         [IB_USER_CM_CMD_LISTEN]        = ib_ucm_listen,
1221         [IB_USER_CM_CMD_ESTABLISH]     = ib_ucm_establish,
1222         [IB_USER_CM_CMD_SEND_REQ]      = ib_ucm_send_req,
1223         [IB_USER_CM_CMD_SEND_REP]      = ib_ucm_send_rep,
1224         [IB_USER_CM_CMD_SEND_RTU]      = ib_ucm_send_rtu,
1225         [IB_USER_CM_CMD_SEND_DREQ]     = ib_ucm_send_dreq,
1226         [IB_USER_CM_CMD_SEND_DREP]     = ib_ucm_send_drep,
1227         [IB_USER_CM_CMD_SEND_REJ]      = ib_ucm_send_rej,
1228         [IB_USER_CM_CMD_SEND_MRA]      = ib_ucm_send_mra,
1229         [IB_USER_CM_CMD_SEND_LAP]      = ib_ucm_send_lap,
1230         [IB_USER_CM_CMD_SEND_APR]      = ib_ucm_send_apr,
1231         [IB_USER_CM_CMD_SEND_SIDR_REQ] = ib_ucm_send_sidr_req,
1232         [IB_USER_CM_CMD_SEND_SIDR_REP] = ib_ucm_send_sidr_rep,
1233         [IB_USER_CM_CMD_EVENT]         = ib_ucm_event,
1234 };
1235
1236 static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,
1237                             size_t len, loff_t *pos)
1238 {
1239         struct ib_ucm_file *file = filp->private_data;
1240         struct ib_ucm_cmd_hdr hdr;
1241         ssize_t result;
1242
1243         if (len < sizeof(hdr))
1244                 return -EINVAL;
1245
1246         if (copy_from_user(&hdr, buf, sizeof(hdr)))
1247                 return -EFAULT;
1248
1249         ucm_dbg("Write. cmd <%d> in <%d> out <%d> len <%Zu>\n",
1250                 hdr.cmd, hdr.in, hdr.out, len);
1251
1252         if (hdr.cmd < 0 || hdr.cmd >= ARRAY_SIZE(ucm_cmd_table))
1253                 return -EINVAL;
1254
1255         if (hdr.in + sizeof(hdr) > len)
1256                 return -EINVAL;
1257
1258         result = ucm_cmd_table[hdr.cmd](file, buf + sizeof(hdr),
1259                                         hdr.in, hdr.out);
1260         if (!result)
1261                 result = len;
1262
1263         return result;
1264 }
1265
1266 static unsigned int ib_ucm_poll(struct file *filp,
1267                                 struct poll_table_struct *wait)
1268 {
1269         struct ib_ucm_file *file = filp->private_data;
1270         unsigned int mask = 0;
1271
1272         poll_wait(filp, &file->poll_wait, wait);
1273
1274         if (!list_empty(&file->events))
1275                 mask = POLLIN | POLLRDNORM;
1276
1277         return mask;
1278 }
1279
1280 static int ib_ucm_open(struct inode *inode, struct file *filp)
1281 {
1282         struct ib_ucm_file *file;
1283
1284         file = kmalloc(sizeof(*file), GFP_KERNEL);
1285         if (!file)
1286                 return -ENOMEM;
1287
1288         INIT_LIST_HEAD(&file->events);
1289         INIT_LIST_HEAD(&file->ctxs);
1290         init_waitqueue_head(&file->poll_wait);
1291
1292         init_MUTEX(&file->mutex);
1293
1294         filp->private_data = file;
1295         file->filp = filp;
1296
1297         ucm_dbg("Created struct\n");
1298
1299         return 0;
1300 }
1301
1302 static int ib_ucm_close(struct inode *inode, struct file *filp)
1303 {
1304         struct ib_ucm_file *file = filp->private_data;
1305         struct ib_ucm_context *ctx;
1306
1307         down(&file->mutex);
1308
1309         while (!list_empty(&file->ctxs)) {
1310
1311                 ctx = list_entry(file->ctxs.next,
1312                                  struct ib_ucm_context, file_list);
1313
1314                 up(&ctx->file->mutex);
1315                 ib_ucm_ctx_put(ctx); /* user reference */
1316                 down(&file->mutex);
1317         }
1318
1319         up(&file->mutex);
1320
1321         kfree(file);
1322
1323         ucm_dbg("Deleted struct\n");
1324         return 0;
1325 }
1326
1327 static struct file_operations ib_ucm_fops = {
1328         .owner   = THIS_MODULE,
1329         .open    = ib_ucm_open,
1330         .release = ib_ucm_close,
1331         .write   = ib_ucm_write,
1332         .poll    = ib_ucm_poll,
1333 };
1334
1335
1336 static struct class *ib_ucm_class;
1337 static struct cdev        ib_ucm_cdev;
1338
1339 static int __init ib_ucm_init(void)
1340 {
1341         int result;
1342
1343         result = register_chrdev_region(IB_UCM_DEV, 1, "infiniband_cm");
1344         if (result) {
1345                 ucm_dbg("Error <%d> registering dev\n", result);
1346                 goto err_chr;
1347         }
1348
1349         cdev_init(&ib_ucm_cdev, &ib_ucm_fops);
1350
1351         result = cdev_add(&ib_ucm_cdev, IB_UCM_DEV, 1);
1352         if (result) {
1353                 ucm_dbg("Error <%d> adding cdev\n", result);
1354                 goto err_cdev;
1355         }
1356
1357         ib_ucm_class = class_create(THIS_MODULE, "infiniband_cm");
1358         if (IS_ERR(ib_ucm_class)) {
1359                 result = PTR_ERR(ib_ucm_class);
1360                 ucm_dbg("Error <%d> creating class\n", result);
1361                 goto err_class;
1362         }
1363
1364         class_device_create(ib_ucm_class, IB_UCM_DEV, NULL, "ucm");
1365
1366         idr_init(&ctx_id_table);
1367         init_MUTEX(&ctx_id_mutex);
1368
1369         return 0;
1370 err_class:
1371         cdev_del(&ib_ucm_cdev);
1372 err_cdev:
1373         unregister_chrdev_region(IB_UCM_DEV, 1);
1374 err_chr:
1375         return result;
1376 }
1377
1378 static void __exit ib_ucm_cleanup(void)
1379 {
1380         class_device_destroy(ib_ucm_class, IB_UCM_DEV);
1381         class_destroy(ib_ucm_class);
1382         cdev_del(&ib_ucm_cdev);
1383         unregister_chrdev_region(IB_UCM_DEV, 1);
1384 }
1385
1386 module_init(ib_ucm_init);
1387 module_exit(ib_ucm_cleanup);