mm: Remove slab destructors from kmem_cache_create().
[linux-2.6.git] / drivers / infiniband / hw / ehca / ehca_mrmw.c
1 /*
2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
3  *
4  *  MR/MW functions
5  *
6  *  Authors: Dietmar Decker <ddecker@de.ibm.com>
7  *           Christoph Raisch <raisch@de.ibm.com>
8  *
9  *  Copyright (c) 2005 IBM Corporation
10  *
11  *  All rights reserved.
12  *
13  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
14  *  BSD.
15  *
16  * OpenIB BSD License
17  *
18  * Redistribution and use in source and binary forms, with or without
19  * modification, are permitted provided that the following conditions are met:
20  *
21  * Redistributions of source code must retain the above copyright notice, this
22  * list of conditions and the following disclaimer.
23  *
24  * Redistributions in binary form must reproduce the above copyright notice,
25  * this list of conditions and the following disclaimer in the documentation
26  * and/or other materials
27  * provided with the distribution.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
36  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
37  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39  * POSSIBILITY OF SUCH DAMAGE.
40  */
41
42 #include <rdma/ib_umem.h>
43
44 #include <asm/current.h>
45
46 #include "ehca_iverbs.h"
47 #include "ehca_mrmw.h"
48 #include "hcp_if.h"
49 #include "hipz_hw.h"
50
51 #define NUM_CHUNKS(length, chunk_size) \
52         (((length) + (chunk_size - 1)) / (chunk_size))
53 /* max number of rpages (per hcall register_rpages) */
54 #define MAX_RPAGES 512
55
56 static struct kmem_cache *mr_cache;
57 static struct kmem_cache *mw_cache;
58
59 static struct ehca_mr *ehca_mr_new(void)
60 {
61         struct ehca_mr *me;
62
63         me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
64         if (me)
65                 spin_lock_init(&me->mrlock);
66         else
67                 ehca_gen_err("alloc failed");
68
69         return me;
70 }
71
72 static void ehca_mr_delete(struct ehca_mr *me)
73 {
74         kmem_cache_free(mr_cache, me);
75 }
76
77 static struct ehca_mw *ehca_mw_new(void)
78 {
79         struct ehca_mw *me;
80
81         me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
82         if (me)
83                 spin_lock_init(&me->mwlock);
84         else
85                 ehca_gen_err("alloc failed");
86
87         return me;
88 }
89
90 static void ehca_mw_delete(struct ehca_mw *me)
91 {
92         kmem_cache_free(mw_cache, me);
93 }
94
95 /*----------------------------------------------------------------------*/
96
97 struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
98 {
99         struct ib_mr *ib_mr;
100         int ret;
101         struct ehca_mr *e_maxmr;
102         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
103         struct ehca_shca *shca =
104                 container_of(pd->device, struct ehca_shca, ib_device);
105
106         if (shca->maxmr) {
107                 e_maxmr = ehca_mr_new();
108                 if (!e_maxmr) {
109                         ehca_err(&shca->ib_device, "out of memory");
110                         ib_mr = ERR_PTR(-ENOMEM);
111                         goto get_dma_mr_exit0;
112                 }
113
114                 ret = ehca_reg_maxmr(shca, e_maxmr, (u64 *)KERNELBASE,
115                                      mr_access_flags, e_pd,
116                                      &e_maxmr->ib.ib_mr.lkey,
117                                      &e_maxmr->ib.ib_mr.rkey);
118                 if (ret) {
119                         ehca_mr_delete(e_maxmr);
120                         ib_mr = ERR_PTR(ret);
121                         goto get_dma_mr_exit0;
122                 }
123                 ib_mr = &e_maxmr->ib.ib_mr;
124         } else {
125                 ehca_err(&shca->ib_device, "no internal max-MR exist!");
126                 ib_mr = ERR_PTR(-EINVAL);
127                 goto get_dma_mr_exit0;
128         }
129
130 get_dma_mr_exit0:
131         if (IS_ERR(ib_mr))
132                 ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ",
133                          PTR_ERR(ib_mr), pd, mr_access_flags);
134         return ib_mr;
135 } /* end ehca_get_dma_mr() */
136
137 /*----------------------------------------------------------------------*/
138
139 struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
140                                struct ib_phys_buf *phys_buf_array,
141                                int num_phys_buf,
142                                int mr_access_flags,
143                                u64 *iova_start)
144 {
145         struct ib_mr *ib_mr;
146         int ret;
147         struct ehca_mr *e_mr;
148         struct ehca_shca *shca =
149                 container_of(pd->device, struct ehca_shca, ib_device);
150         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
151
152         u64 size;
153
154         if ((num_phys_buf <= 0) || !phys_buf_array) {
155                 ehca_err(pd->device, "bad input values: num_phys_buf=%x "
156                          "phys_buf_array=%p", num_phys_buf, phys_buf_array);
157                 ib_mr = ERR_PTR(-EINVAL);
158                 goto reg_phys_mr_exit0;
159         }
160         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
161              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
162             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
163              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
164                 /*
165                  * Remote Write Access requires Local Write Access
166                  * Remote Atomic Access requires Local Write Access
167                  */
168                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
169                          mr_access_flags);
170                 ib_mr = ERR_PTR(-EINVAL);
171                 goto reg_phys_mr_exit0;
172         }
173
174         /* check physical buffer list and calculate size */
175         ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
176                                             iova_start, &size);
177         if (ret) {
178                 ib_mr = ERR_PTR(ret);
179                 goto reg_phys_mr_exit0;
180         }
181         if ((size == 0) ||
182             (((u64)iova_start + size) < (u64)iova_start)) {
183                 ehca_err(pd->device, "bad input values: size=%lx iova_start=%p",
184                          size, iova_start);
185                 ib_mr = ERR_PTR(-EINVAL);
186                 goto reg_phys_mr_exit0;
187         }
188
189         e_mr = ehca_mr_new();
190         if (!e_mr) {
191                 ehca_err(pd->device, "out of memory");
192                 ib_mr = ERR_PTR(-ENOMEM);
193                 goto reg_phys_mr_exit0;
194         }
195
196         /* register MR on HCA */
197         if (ehca_mr_is_maxmr(size, iova_start)) {
198                 e_mr->flags |= EHCA_MR_FLAG_MAXMR;
199                 ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
200                                      e_pd, &e_mr->ib.ib_mr.lkey,
201                                      &e_mr->ib.ib_mr.rkey);
202                 if (ret) {
203                         ib_mr = ERR_PTR(ret);
204                         goto reg_phys_mr_exit1;
205                 }
206         } else {
207                 struct ehca_mr_pginfo pginfo;
208                 u32 num_kpages;
209                 u32 num_hwpages;
210
211                 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
212                                         PAGE_SIZE);
213                 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) +
214                                          size, EHCA_PAGESIZE);
215                 memset(&pginfo, 0, sizeof(pginfo));
216                 pginfo.type = EHCA_MR_PGI_PHYS;
217                 pginfo.num_kpages = num_kpages;
218                 pginfo.num_hwpages = num_hwpages;
219                 pginfo.u.phy.num_phys_buf = num_phys_buf;
220                 pginfo.u.phy.phys_buf_array = phys_buf_array;
221                 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
222                                       EHCA_PAGESIZE);
223
224                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
225                                   e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
226                                   &e_mr->ib.ib_mr.rkey);
227                 if (ret) {
228                         ib_mr = ERR_PTR(ret);
229                         goto reg_phys_mr_exit1;
230                 }
231         }
232
233         /* successful registration of all pages */
234         return &e_mr->ib.ib_mr;
235
236 reg_phys_mr_exit1:
237         ehca_mr_delete(e_mr);
238 reg_phys_mr_exit0:
239         if (IS_ERR(ib_mr))
240                 ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p "
241                          "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
242                          PTR_ERR(ib_mr), pd, phys_buf_array,
243                          num_phys_buf, mr_access_flags, iova_start);
244         return ib_mr;
245 } /* end ehca_reg_phys_mr() */
246
247 /*----------------------------------------------------------------------*/
248
249 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
250                                u64 virt, int mr_access_flags,
251                                struct ib_udata *udata)
252 {
253         struct ib_mr *ib_mr;
254         struct ehca_mr *e_mr;
255         struct ehca_shca *shca =
256                 container_of(pd->device, struct ehca_shca, ib_device);
257         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
258         struct ehca_mr_pginfo pginfo;
259         int ret;
260         u32 num_kpages;
261         u32 num_hwpages;
262
263         if (!pd) {
264                 ehca_gen_err("bad pd=%p", pd);
265                 return ERR_PTR(-EFAULT);
266         }
267
268         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
269              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
270             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
271              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
272                 /*
273                  * Remote Write Access requires Local Write Access
274                  * Remote Atomic Access requires Local Write Access
275                  */
276                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
277                          mr_access_flags);
278                 ib_mr = ERR_PTR(-EINVAL);
279                 goto reg_user_mr_exit0;
280         }
281
282         if (length == 0 || virt + length < virt) {
283                 ehca_err(pd->device, "bad input values: length=%lx "
284                          "virt_base=%lx", length, virt);
285                 ib_mr = ERR_PTR(-EINVAL);
286                 goto reg_user_mr_exit0;
287         }
288
289         e_mr = ehca_mr_new();
290         if (!e_mr) {
291                 ehca_err(pd->device, "out of memory");
292                 ib_mr = ERR_PTR(-ENOMEM);
293                 goto reg_user_mr_exit0;
294         }
295
296         e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
297                                  mr_access_flags);
298         if (IS_ERR(e_mr->umem)) {
299                 ib_mr = (void *)e_mr->umem;
300                 goto reg_user_mr_exit1;
301         }
302
303         if (e_mr->umem->page_size != PAGE_SIZE) {
304                 ehca_err(pd->device, "page size not supported, "
305                          "e_mr->umem->page_size=%x", e_mr->umem->page_size);
306                 ib_mr = ERR_PTR(-EINVAL);
307                 goto reg_user_mr_exit2;
308         }
309
310         /* determine number of MR pages */
311         num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
312         num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length,
313                                  EHCA_PAGESIZE);
314
315         /* register MR on HCA */
316         memset(&pginfo, 0, sizeof(pginfo));
317         pginfo.type = EHCA_MR_PGI_USER;
318         pginfo.num_kpages = num_kpages;
319         pginfo.num_hwpages = num_hwpages;
320         pginfo.u.usr.region = e_mr->umem;
321         pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE;
322         pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk,
323                                                      (&e_mr->umem->chunk_list),
324                                                      list);
325
326         ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
327                           e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
328                           &e_mr->ib.ib_mr.rkey);
329         if (ret) {
330                 ib_mr = ERR_PTR(ret);
331                 goto reg_user_mr_exit2;
332         }
333
334         /* successful registration of all pages */
335         return &e_mr->ib.ib_mr;
336
337 reg_user_mr_exit2:
338         ib_umem_release(e_mr->umem);
339 reg_user_mr_exit1:
340         ehca_mr_delete(e_mr);
341 reg_user_mr_exit0:
342         if (IS_ERR(ib_mr))
343                 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x"
344                          " udata=%p",
345                          PTR_ERR(ib_mr), pd, mr_access_flags, udata);
346         return ib_mr;
347 } /* end ehca_reg_user_mr() */
348
349 /*----------------------------------------------------------------------*/
350
351 int ehca_rereg_phys_mr(struct ib_mr *mr,
352                        int mr_rereg_mask,
353                        struct ib_pd *pd,
354                        struct ib_phys_buf *phys_buf_array,
355                        int num_phys_buf,
356                        int mr_access_flags,
357                        u64 *iova_start)
358 {
359         int ret;
360
361         struct ehca_shca *shca =
362                 container_of(mr->device, struct ehca_shca, ib_device);
363         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
364         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
365         u64 new_size;
366         u64 *new_start;
367         u32 new_acl;
368         struct ehca_pd *new_pd;
369         u32 tmp_lkey, tmp_rkey;
370         unsigned long sl_flags;
371         u32 num_kpages = 0;
372         u32 num_hwpages = 0;
373         struct ehca_mr_pginfo pginfo;
374         u32 cur_pid = current->tgid;
375
376         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
377             (my_pd->ownpid != cur_pid)) {
378                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
379                          cur_pid, my_pd->ownpid);
380                 ret = -EINVAL;
381                 goto rereg_phys_mr_exit0;
382         }
383
384         if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
385                 /* TODO not supported, because PHYP rereg hCall needs pages */
386                 ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
387                          "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
388                 ret = -EINVAL;
389                 goto rereg_phys_mr_exit0;
390         }
391
392         if (mr_rereg_mask & IB_MR_REREG_PD) {
393                 if (!pd) {
394                         ehca_err(mr->device, "rereg with bad pd, pd=%p "
395                                  "mr_rereg_mask=%x", pd, mr_rereg_mask);
396                         ret = -EINVAL;
397                         goto rereg_phys_mr_exit0;
398                 }
399         }
400
401         if ((mr_rereg_mask &
402              ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
403             (mr_rereg_mask == 0)) {
404                 ret = -EINVAL;
405                 goto rereg_phys_mr_exit0;
406         }
407
408         /* check other parameters */
409         if (e_mr == shca->maxmr) {
410                 /* should be impossible, however reject to be sure */
411                 ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
412                          "shca->maxmr=%p mr->lkey=%x",
413                          mr, shca->maxmr, mr->lkey);
414                 ret = -EINVAL;
415                 goto rereg_phys_mr_exit0;
416         }
417         if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
418                 if (e_mr->flags & EHCA_MR_FLAG_FMR) {
419                         ehca_err(mr->device, "not supported for FMR, mr=%p "
420                                  "flags=%x", mr, e_mr->flags);
421                         ret = -EINVAL;
422                         goto rereg_phys_mr_exit0;
423                 }
424                 if (!phys_buf_array || num_phys_buf <= 0) {
425                         ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
426                                  " phys_buf_array=%p num_phys_buf=%x",
427                                  mr_rereg_mask, phys_buf_array, num_phys_buf);
428                         ret = -EINVAL;
429                         goto rereg_phys_mr_exit0;
430                 }
431         }
432         if ((mr_rereg_mask & IB_MR_REREG_ACCESS) &&     /* change ACL */
433             (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
434               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
435              ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
436               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
437                 /*
438                  * Remote Write Access requires Local Write Access
439                  * Remote Atomic Access requires Local Write Access
440                  */
441                 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
442                          "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
443                 ret = -EINVAL;
444                 goto rereg_phys_mr_exit0;
445         }
446
447         /* set requested values dependent on rereg request */
448         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
449         new_start = e_mr->start;
450         new_size = e_mr->size;
451         new_acl = e_mr->acl;
452         new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
453
454         if (mr_rereg_mask & IB_MR_REREG_TRANS) {
455                 new_start = iova_start; /* change address */
456                 /* check physical buffer list and calculate size */
457                 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
458                                                     num_phys_buf, iova_start,
459                                                     &new_size);
460                 if (ret)
461                         goto rereg_phys_mr_exit1;
462                 if ((new_size == 0) ||
463                     (((u64)iova_start + new_size) < (u64)iova_start)) {
464                         ehca_err(mr->device, "bad input values: new_size=%lx "
465                                  "iova_start=%p", new_size, iova_start);
466                         ret = -EINVAL;
467                         goto rereg_phys_mr_exit1;
468                 }
469                 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
470                                         new_size, PAGE_SIZE);
471                 num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) +
472                                          new_size, EHCA_PAGESIZE);
473                 memset(&pginfo, 0, sizeof(pginfo));
474                 pginfo.type = EHCA_MR_PGI_PHYS;
475                 pginfo.num_kpages = num_kpages;
476                 pginfo.num_hwpages = num_hwpages;
477                 pginfo.u.phy.num_phys_buf = num_phys_buf;
478                 pginfo.u.phy.phys_buf_array = phys_buf_array;
479                 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
480                                       EHCA_PAGESIZE);
481         }
482         if (mr_rereg_mask & IB_MR_REREG_ACCESS)
483                 new_acl = mr_access_flags;
484         if (mr_rereg_mask & IB_MR_REREG_PD)
485                 new_pd = container_of(pd, struct ehca_pd, ib_pd);
486
487         ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
488                             new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
489         if (ret)
490                 goto rereg_phys_mr_exit1;
491
492         /* successful reregistration */
493         if (mr_rereg_mask & IB_MR_REREG_PD)
494                 mr->pd = pd;
495         mr->lkey = tmp_lkey;
496         mr->rkey = tmp_rkey;
497
498 rereg_phys_mr_exit1:
499         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
500 rereg_phys_mr_exit0:
501         if (ret)
502                 ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
503                          "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
504                          "iova_start=%p",
505                          ret, mr, mr_rereg_mask, pd, phys_buf_array,
506                          num_phys_buf, mr_access_flags, iova_start);
507         return ret;
508 } /* end ehca_rereg_phys_mr() */
509
510 /*----------------------------------------------------------------------*/
511
512 int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
513 {
514         int ret = 0;
515         u64 h_ret;
516         struct ehca_shca *shca =
517                 container_of(mr->device, struct ehca_shca, ib_device);
518         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
519         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
520         u32 cur_pid = current->tgid;
521         unsigned long sl_flags;
522         struct ehca_mr_hipzout_parms hipzout;
523
524         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
525             (my_pd->ownpid != cur_pid)) {
526                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
527                          cur_pid, my_pd->ownpid);
528                 ret = -EINVAL;
529                 goto query_mr_exit0;
530         }
531
532         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
533                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
534                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
535                 ret = -EINVAL;
536                 goto query_mr_exit0;
537         }
538
539         memset(mr_attr, 0, sizeof(struct ib_mr_attr));
540         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
541
542         h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
543         if (h_ret != H_SUCCESS) {
544                 ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p "
545                          "hca_hndl=%lx mr_hndl=%lx lkey=%x",
546                          h_ret, mr, shca->ipz_hca_handle.handle,
547                          e_mr->ipz_mr_handle.handle, mr->lkey);
548                 ret = ehca2ib_return_code(h_ret);
549                 goto query_mr_exit1;
550         }
551         mr_attr->pd = mr->pd;
552         mr_attr->device_virt_addr = hipzout.vaddr;
553         mr_attr->size = hipzout.len;
554         mr_attr->lkey = hipzout.lkey;
555         mr_attr->rkey = hipzout.rkey;
556         ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
557
558 query_mr_exit1:
559         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
560 query_mr_exit0:
561         if (ret)
562                 ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p",
563                          ret, mr, mr_attr);
564         return ret;
565 } /* end ehca_query_mr() */
566
567 /*----------------------------------------------------------------------*/
568
569 int ehca_dereg_mr(struct ib_mr *mr)
570 {
571         int ret = 0;
572         u64 h_ret;
573         struct ehca_shca *shca =
574                 container_of(mr->device, struct ehca_shca, ib_device);
575         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
576         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
577         u32 cur_pid = current->tgid;
578
579         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
580             (my_pd->ownpid != cur_pid)) {
581                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
582                          cur_pid, my_pd->ownpid);
583                 ret = -EINVAL;
584                 goto dereg_mr_exit0;
585         }
586
587         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
588                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
589                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
590                 ret = -EINVAL;
591                 goto dereg_mr_exit0;
592         } else if (e_mr == shca->maxmr) {
593                 /* should be impossible, however reject to be sure */
594                 ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
595                          "shca->maxmr=%p mr->lkey=%x",
596                          mr, shca->maxmr, mr->lkey);
597                 ret = -EINVAL;
598                 goto dereg_mr_exit0;
599         }
600
601         /* TODO: BUSY: MR still has bound window(s) */
602         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
603         if (h_ret != H_SUCCESS) {
604                 ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p "
605                          "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
606                          h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
607                          e_mr->ipz_mr_handle.handle, mr->lkey);
608                 ret = ehca2ib_return_code(h_ret);
609                 goto dereg_mr_exit0;
610         }
611
612         if (e_mr->umem)
613                 ib_umem_release(e_mr->umem);
614
615         /* successful deregistration */
616         ehca_mr_delete(e_mr);
617
618 dereg_mr_exit0:
619         if (ret)
620                 ehca_err(mr->device, "ret=%x mr=%p", ret, mr);
621         return ret;
622 } /* end ehca_dereg_mr() */
623
624 /*----------------------------------------------------------------------*/
625
626 struct ib_mw *ehca_alloc_mw(struct ib_pd *pd)
627 {
628         struct ib_mw *ib_mw;
629         u64 h_ret;
630         struct ehca_mw *e_mw;
631         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
632         struct ehca_shca *shca =
633                 container_of(pd->device, struct ehca_shca, ib_device);
634         struct ehca_mw_hipzout_parms hipzout;
635
636         e_mw = ehca_mw_new();
637         if (!e_mw) {
638                 ib_mw = ERR_PTR(-ENOMEM);
639                 goto alloc_mw_exit0;
640         }
641
642         h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
643                                          e_pd->fw_pd, &hipzout);
644         if (h_ret != H_SUCCESS) {
645                 ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx "
646                          "shca=%p hca_hndl=%lx mw=%p",
647                          h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
648                 ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
649                 goto alloc_mw_exit1;
650         }
651         /* successful MW allocation */
652         e_mw->ipz_mw_handle = hipzout.handle;
653         e_mw->ib_mw.rkey    = hipzout.rkey;
654         return &e_mw->ib_mw;
655
656 alloc_mw_exit1:
657         ehca_mw_delete(e_mw);
658 alloc_mw_exit0:
659         if (IS_ERR(ib_mw))
660                 ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd);
661         return ib_mw;
662 } /* end ehca_alloc_mw() */
663
664 /*----------------------------------------------------------------------*/
665
666 int ehca_bind_mw(struct ib_qp *qp,
667                  struct ib_mw *mw,
668                  struct ib_mw_bind *mw_bind)
669 {
670         /* TODO: not supported up to now */
671         ehca_gen_err("bind MW currently not supported by HCAD");
672
673         return -EPERM;
674 } /* end ehca_bind_mw() */
675
676 /*----------------------------------------------------------------------*/
677
678 int ehca_dealloc_mw(struct ib_mw *mw)
679 {
680         u64 h_ret;
681         struct ehca_shca *shca =
682                 container_of(mw->device, struct ehca_shca, ib_device);
683         struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
684
685         h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
686         if (h_ret != H_SUCCESS) {
687                 ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p "
688                          "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
689                          h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
690                          e_mw->ipz_mw_handle.handle);
691                 return ehca2ib_return_code(h_ret);
692         }
693         /* successful deallocation */
694         ehca_mw_delete(e_mw);
695         return 0;
696 } /* end ehca_dealloc_mw() */
697
698 /*----------------------------------------------------------------------*/
699
700 struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
701                               int mr_access_flags,
702                               struct ib_fmr_attr *fmr_attr)
703 {
704         struct ib_fmr *ib_fmr;
705         struct ehca_shca *shca =
706                 container_of(pd->device, struct ehca_shca, ib_device);
707         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
708         struct ehca_mr *e_fmr;
709         int ret;
710         u32 tmp_lkey, tmp_rkey;
711         struct ehca_mr_pginfo pginfo;
712
713         /* check other parameters */
714         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
715              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
716             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
717              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
718                 /*
719                  * Remote Write Access requires Local Write Access
720                  * Remote Atomic Access requires Local Write Access
721                  */
722                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
723                          mr_access_flags);
724                 ib_fmr = ERR_PTR(-EINVAL);
725                 goto alloc_fmr_exit0;
726         }
727         if (mr_access_flags & IB_ACCESS_MW_BIND) {
728                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
729                          mr_access_flags);
730                 ib_fmr = ERR_PTR(-EINVAL);
731                 goto alloc_fmr_exit0;
732         }
733         if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
734                 ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
735                          "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
736                          fmr_attr->max_pages, fmr_attr->max_maps,
737                          fmr_attr->page_shift);
738                 ib_fmr = ERR_PTR(-EINVAL);
739                 goto alloc_fmr_exit0;
740         }
741         if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
742             ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
743                 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
744                          fmr_attr->page_shift);
745                 ib_fmr = ERR_PTR(-EINVAL);
746                 goto alloc_fmr_exit0;
747         }
748
749         e_fmr = ehca_mr_new();
750         if (!e_fmr) {
751                 ib_fmr = ERR_PTR(-ENOMEM);
752                 goto alloc_fmr_exit0;
753         }
754         e_fmr->flags |= EHCA_MR_FLAG_FMR;
755
756         /* register MR on HCA */
757         memset(&pginfo, 0, sizeof(pginfo));
758         ret = ehca_reg_mr(shca, e_fmr, NULL,
759                           fmr_attr->max_pages * (1 << fmr_attr->page_shift),
760                           mr_access_flags, e_pd, &pginfo,
761                           &tmp_lkey, &tmp_rkey);
762         if (ret) {
763                 ib_fmr = ERR_PTR(ret);
764                 goto alloc_fmr_exit1;
765         }
766
767         /* successful */
768         e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
769         e_fmr->fmr_max_pages = fmr_attr->max_pages;
770         e_fmr->fmr_max_maps = fmr_attr->max_maps;
771         e_fmr->fmr_map_cnt = 0;
772         return &e_fmr->ib.ib_fmr;
773
774 alloc_fmr_exit1:
775         ehca_mr_delete(e_fmr);
776 alloc_fmr_exit0:
777         if (IS_ERR(ib_fmr))
778                 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x "
779                          "fmr_attr=%p", PTR_ERR(ib_fmr), pd,
780                          mr_access_flags, fmr_attr);
781         return ib_fmr;
782 } /* end ehca_alloc_fmr() */
783
784 /*----------------------------------------------------------------------*/
785
786 int ehca_map_phys_fmr(struct ib_fmr *fmr,
787                       u64 *page_list,
788                       int list_len,
789                       u64 iova)
790 {
791         int ret;
792         struct ehca_shca *shca =
793                 container_of(fmr->device, struct ehca_shca, ib_device);
794         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
795         struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
796         struct ehca_mr_pginfo pginfo;
797         u32 tmp_lkey, tmp_rkey;
798
799         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
800                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
801                          e_fmr, e_fmr->flags);
802                 ret = -EINVAL;
803                 goto map_phys_fmr_exit0;
804         }
805         ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
806         if (ret)
807                 goto map_phys_fmr_exit0;
808         if (iova % e_fmr->fmr_page_size) {
809                 /* only whole-numbered pages */
810                 ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x",
811                          iova, e_fmr->fmr_page_size);
812                 ret = -EINVAL;
813                 goto map_phys_fmr_exit0;
814         }
815         if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
816                 /* HCAD does not limit the maps, however trace this anyway */
817                 ehca_info(fmr->device, "map limit exceeded, fmr=%p "
818                           "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
819                           fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
820         }
821
822         memset(&pginfo, 0, sizeof(pginfo));
823         pginfo.type = EHCA_MR_PGI_FMR;
824         pginfo.num_kpages = list_len;
825         pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
826         pginfo.u.fmr.page_list = page_list;
827         pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) /
828                               EHCA_PAGESIZE);
829         pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
830
831         ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
832                             list_len * e_fmr->fmr_page_size,
833                             e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
834         if (ret)
835                 goto map_phys_fmr_exit0;
836
837         /* successful reregistration */
838         e_fmr->fmr_map_cnt++;
839         e_fmr->ib.ib_fmr.lkey = tmp_lkey;
840         e_fmr->ib.ib_fmr.rkey = tmp_rkey;
841         return 0;
842
843 map_phys_fmr_exit0:
844         if (ret)
845                 ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x "
846                          "iova=%lx", ret, fmr, page_list, list_len, iova);
847         return ret;
848 } /* end ehca_map_phys_fmr() */
849
850 /*----------------------------------------------------------------------*/
851
852 int ehca_unmap_fmr(struct list_head *fmr_list)
853 {
854         int ret = 0;
855         struct ib_fmr *ib_fmr;
856         struct ehca_shca *shca = NULL;
857         struct ehca_shca *prev_shca;
858         struct ehca_mr *e_fmr;
859         u32 num_fmr = 0;
860         u32 unmap_fmr_cnt = 0;
861
862         /* check all FMR belong to same SHCA, and check internal flag */
863         list_for_each_entry(ib_fmr, fmr_list, list) {
864                 prev_shca = shca;
865                 if (!ib_fmr) {
866                         ehca_gen_err("bad fmr=%p in list", ib_fmr);
867                         ret = -EINVAL;
868                         goto unmap_fmr_exit0;
869                 }
870                 shca = container_of(ib_fmr->device, struct ehca_shca,
871                                     ib_device);
872                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
873                 if ((shca != prev_shca) && prev_shca) {
874                         ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
875                                  "prev_shca=%p e_fmr=%p",
876                                  shca, prev_shca, e_fmr);
877                         ret = -EINVAL;
878                         goto unmap_fmr_exit0;
879                 }
880                 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
881                         ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
882                                  "e_fmr->flags=%x", e_fmr, e_fmr->flags);
883                         ret = -EINVAL;
884                         goto unmap_fmr_exit0;
885                 }
886                 num_fmr++;
887         }
888
889         /* loop over all FMRs to unmap */
890         list_for_each_entry(ib_fmr, fmr_list, list) {
891                 unmap_fmr_cnt++;
892                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
893                 shca = container_of(ib_fmr->device, struct ehca_shca,
894                                     ib_device);
895                 ret = ehca_unmap_one_fmr(shca, e_fmr);
896                 if (ret) {
897                         /* unmap failed, stop unmapping of rest of FMRs */
898                         ehca_err(&shca->ib_device, "unmap of one FMR failed, "
899                                  "stop rest, e_fmr=%p num_fmr=%x "
900                                  "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
901                                  unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
902                         goto unmap_fmr_exit0;
903                 }
904         }
905
906 unmap_fmr_exit0:
907         if (ret)
908                 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
909                              ret, fmr_list, num_fmr, unmap_fmr_cnt);
910         return ret;
911 } /* end ehca_unmap_fmr() */
912
913 /*----------------------------------------------------------------------*/
914
915 int ehca_dealloc_fmr(struct ib_fmr *fmr)
916 {
917         int ret;
918         u64 h_ret;
919         struct ehca_shca *shca =
920                 container_of(fmr->device, struct ehca_shca, ib_device);
921         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
922
923         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
924                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
925                          e_fmr, e_fmr->flags);
926                 ret = -EINVAL;
927                 goto free_fmr_exit0;
928         }
929
930         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
931         if (h_ret != H_SUCCESS) {
932                 ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
933                          "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
934                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
935                          e_fmr->ipz_mr_handle.handle, fmr->lkey);
936                 ret = ehca2ib_return_code(h_ret);
937                 goto free_fmr_exit0;
938         }
939         /* successful deregistration */
940         ehca_mr_delete(e_fmr);
941         return 0;
942
943 free_fmr_exit0:
944         if (ret)
945                 ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr);
946         return ret;
947 } /* end ehca_dealloc_fmr() */
948
949 /*----------------------------------------------------------------------*/
950
951 int ehca_reg_mr(struct ehca_shca *shca,
952                 struct ehca_mr *e_mr,
953                 u64 *iova_start,
954                 u64 size,
955                 int acl,
956                 struct ehca_pd *e_pd,
957                 struct ehca_mr_pginfo *pginfo,
958                 u32 *lkey, /*OUT*/
959                 u32 *rkey) /*OUT*/
960 {
961         int ret;
962         u64 h_ret;
963         u32 hipz_acl;
964         struct ehca_mr_hipzout_parms hipzout;
965
966         ehca_mrmw_map_acl(acl, &hipz_acl);
967         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
968         if (ehca_use_hp_mr == 1)
969                 hipz_acl |= 0x00000001;
970
971         h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
972                                          (u64)iova_start, size, hipz_acl,
973                                          e_pd->fw_pd, &hipzout);
974         if (h_ret != H_SUCCESS) {
975                 ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx "
976                          "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle);
977                 ret = ehca2ib_return_code(h_ret);
978                 goto ehca_reg_mr_exit0;
979         }
980
981         e_mr->ipz_mr_handle = hipzout.handle;
982
983         ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
984         if (ret)
985                 goto ehca_reg_mr_exit1;
986
987         /* successful registration */
988         e_mr->num_kpages = pginfo->num_kpages;
989         e_mr->num_hwpages = pginfo->num_hwpages;
990         e_mr->start = iova_start;
991         e_mr->size = size;
992         e_mr->acl = acl;
993         *lkey = hipzout.lkey;
994         *rkey = hipzout.rkey;
995         return 0;
996
997 ehca_reg_mr_exit1:
998         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
999         if (h_ret != H_SUCCESS) {
1000                 ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p "
1001                          "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
1002                          "pginfo=%p num_kpages=%lx num_hwpages=%lx ret=%x",
1003                          h_ret, shca, e_mr, iova_start, size, acl, e_pd,
1004                          hipzout.lkey, pginfo, pginfo->num_kpages,
1005                          pginfo->num_hwpages, ret);
1006                 ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
1007                          "not recoverable");
1008         }
1009 ehca_reg_mr_exit0:
1010         if (ret)
1011                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1012                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1013                          "num_kpages=%lx num_hwpages=%lx",
1014                          ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
1015                          pginfo->num_kpages, pginfo->num_hwpages);
1016         return ret;
1017 } /* end ehca_reg_mr() */
1018
1019 /*----------------------------------------------------------------------*/
1020
1021 int ehca_reg_mr_rpages(struct ehca_shca *shca,
1022                        struct ehca_mr *e_mr,
1023                        struct ehca_mr_pginfo *pginfo)
1024 {
1025         int ret = 0;
1026         u64 h_ret;
1027         u32 rnum;
1028         u64 rpage;
1029         u32 i;
1030         u64 *kpage;
1031
1032         kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1033         if (!kpage) {
1034                 ehca_err(&shca->ib_device, "kpage alloc failed");
1035                 ret = -ENOMEM;
1036                 goto ehca_reg_mr_rpages_exit0;
1037         }
1038
1039         /* max 512 pages per shot */
1040         for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
1041
1042                 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1043                         rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
1044                         if (rnum == 0)
1045                                 rnum = MAX_RPAGES;      /* last shot is full */
1046                 } else
1047                         rnum = MAX_RPAGES;
1048
1049                 ret = ehca_set_pagebuf(pginfo, rnum, kpage);
1050                 if (ret) {
1051                         ehca_err(&shca->ib_device, "ehca_set_pagebuf "
1052                                          "bad rc, ret=%x rnum=%x kpage=%p",
1053                                          ret, rnum, kpage);
1054                         goto ehca_reg_mr_rpages_exit1;
1055                 }
1056
1057                 if (rnum > 1) {
1058                         rpage = virt_to_abs(kpage);
1059                         if (!rpage) {
1060                                 ehca_err(&shca->ib_device, "kpage=%p i=%x",
1061                                          kpage, i);
1062                                 ret = -EFAULT;
1063                                 goto ehca_reg_mr_rpages_exit1;
1064                         }
1065                 } else
1066                         rpage = *kpage;
1067
1068                 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
1069                                                  0, /* pagesize 4k */
1070                                                  0, rpage, rnum);
1071
1072                 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1073                         /*
1074                          * check for 'registration complete'==H_SUCCESS
1075                          * and for 'page registered'==H_PAGE_REGISTERED
1076                          */
1077                         if (h_ret != H_SUCCESS) {
1078                                 ehca_err(&shca->ib_device, "last "
1079                                          "hipz_reg_rpage_mr failed, h_ret=%lx "
1080                                          "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
1081                                          " lkey=%x", h_ret, e_mr, i,
1082                                          shca->ipz_hca_handle.handle,
1083                                          e_mr->ipz_mr_handle.handle,
1084                                          e_mr->ib.ib_mr.lkey);
1085                                 ret = ehca2ib_return_code(h_ret);
1086                                 break;
1087                         } else
1088                                 ret = 0;
1089                 } else if (h_ret != H_PAGE_REGISTERED) {
1090                         ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
1091                                  "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
1092                                  "mr_hndl=%lx", h_ret, e_mr, i,
1093                                  e_mr->ib.ib_mr.lkey,
1094                                  shca->ipz_hca_handle.handle,
1095                                  e_mr->ipz_mr_handle.handle);
1096                         ret = ehca2ib_return_code(h_ret);
1097                         break;
1098                 } else
1099                         ret = 0;
1100         } /* end for(i) */
1101
1102
1103 ehca_reg_mr_rpages_exit1:
1104         ehca_free_fw_ctrlblock(kpage);
1105 ehca_reg_mr_rpages_exit0:
1106         if (ret)
1107                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p "
1108                          "num_kpages=%lx num_hwpages=%lx", ret, shca, e_mr,
1109                          pginfo, pginfo->num_kpages, pginfo->num_hwpages);
1110         return ret;
1111 } /* end ehca_reg_mr_rpages() */
1112
1113 /*----------------------------------------------------------------------*/
1114
1115 inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1116                                 struct ehca_mr *e_mr,
1117                                 u64 *iova_start,
1118                                 u64 size,
1119                                 u32 acl,
1120                                 struct ehca_pd *e_pd,
1121                                 struct ehca_mr_pginfo *pginfo,
1122                                 u32 *lkey, /*OUT*/
1123                                 u32 *rkey) /*OUT*/
1124 {
1125         int ret;
1126         u64 h_ret;
1127         u32 hipz_acl;
1128         u64 *kpage;
1129         u64 rpage;
1130         struct ehca_mr_pginfo pginfo_save;
1131         struct ehca_mr_hipzout_parms hipzout;
1132
1133         ehca_mrmw_map_acl(acl, &hipz_acl);
1134         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1135
1136         kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1137         if (!kpage) {
1138                 ehca_err(&shca->ib_device, "kpage alloc failed");
1139                 ret = -ENOMEM;
1140                 goto ehca_rereg_mr_rereg1_exit0;
1141         }
1142
1143         pginfo_save = *pginfo;
1144         ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
1145         if (ret) {
1146                 ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
1147                          "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx "
1148                          "kpage=%p", e_mr, pginfo, pginfo->type,
1149                          pginfo->num_kpages, pginfo->num_hwpages, kpage);
1150                 goto ehca_rereg_mr_rereg1_exit1;
1151         }
1152         rpage = virt_to_abs(kpage);
1153         if (!rpage) {
1154                 ehca_err(&shca->ib_device, "kpage=%p", kpage);
1155                 ret = -EFAULT;
1156                 goto ehca_rereg_mr_rereg1_exit1;
1157         }
1158         h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
1159                                       (u64)iova_start, size, hipz_acl,
1160                                       e_pd->fw_pd, rpage, &hipzout);
1161         if (h_ret != H_SUCCESS) {
1162                 /*
1163                  * reregistration unsuccessful, try it again with the 3 hCalls,
1164                  * e.g. this is required in case H_MR_CONDITION
1165                  * (MW bound or MR is shared)
1166                  */
1167                 ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
1168                           "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr);
1169                 *pginfo = pginfo_save;
1170                 ret = -EAGAIN;
1171         } else if ((u64 *)hipzout.vaddr != iova_start) {
1172                 ehca_err(&shca->ib_device, "PHYP changed iova_start in "
1173                          "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
1174                          "mr_handle=%lx lkey=%x lkey_out=%x", iova_start,
1175                          hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
1176                          e_mr->ib.ib_mr.lkey, hipzout.lkey);
1177                 ret = -EFAULT;
1178         } else {
1179                 /*
1180                  * successful reregistration
1181                  * note: start and start_out are identical for eServer HCAs
1182                  */
1183                 e_mr->num_kpages = pginfo->num_kpages;
1184                 e_mr->num_hwpages = pginfo->num_hwpages;
1185                 e_mr->start = iova_start;
1186                 e_mr->size = size;
1187                 e_mr->acl = acl;
1188                 *lkey = hipzout.lkey;
1189                 *rkey = hipzout.rkey;
1190         }
1191
1192 ehca_rereg_mr_rereg1_exit1:
1193         ehca_free_fw_ctrlblock(kpage);
1194 ehca_rereg_mr_rereg1_exit0:
1195         if ( ret && (ret != -EAGAIN) )
1196                 ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x "
1197                          "pginfo=%p num_kpages=%lx num_hwpages=%lx",
1198                          ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
1199                          pginfo->num_hwpages);
1200         return ret;
1201 } /* end ehca_rereg_mr_rereg1() */
1202
1203 /*----------------------------------------------------------------------*/
1204
1205 int ehca_rereg_mr(struct ehca_shca *shca,
1206                   struct ehca_mr *e_mr,
1207                   u64 *iova_start,
1208                   u64 size,
1209                   int acl,
1210                   struct ehca_pd *e_pd,
1211                   struct ehca_mr_pginfo *pginfo,
1212                   u32 *lkey,
1213                   u32 *rkey)
1214 {
1215         int ret = 0;
1216         u64 h_ret;
1217         int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
1218         int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
1219
1220         /* first determine reregistration hCall(s) */
1221         if ((pginfo->num_hwpages > MAX_RPAGES) ||
1222             (e_mr->num_hwpages > MAX_RPAGES) ||
1223             (pginfo->num_hwpages > e_mr->num_hwpages)) {
1224                 ehca_dbg(&shca->ib_device, "Rereg3 case, "
1225                          "pginfo->num_hwpages=%lx e_mr->num_hwpages=%x",
1226                          pginfo->num_hwpages, e_mr->num_hwpages);
1227                 rereg_1_hcall = 0;
1228                 rereg_3_hcall = 1;
1229         }
1230
1231         if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
1232                 rereg_1_hcall = 0;
1233                 rereg_3_hcall = 1;
1234                 e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
1235                 ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
1236                          e_mr);
1237         }
1238
1239         if (rereg_1_hcall) {
1240                 ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
1241                                            acl, e_pd, pginfo, lkey, rkey);
1242                 if (ret) {
1243                         if (ret == -EAGAIN)
1244                                 rereg_3_hcall = 1;
1245                         else
1246                                 goto ehca_rereg_mr_exit0;
1247                 }
1248         }
1249
1250         if (rereg_3_hcall) {
1251                 struct ehca_mr save_mr;
1252
1253                 /* first deregister old MR */
1254                 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
1255                 if (h_ret != H_SUCCESS) {
1256                         ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1257                                  "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
1258                                  "mr->lkey=%x",
1259                                  h_ret, e_mr, shca->ipz_hca_handle.handle,
1260                                  e_mr->ipz_mr_handle.handle,
1261                                  e_mr->ib.ib_mr.lkey);
1262                         ret = ehca2ib_return_code(h_ret);
1263                         goto ehca_rereg_mr_exit0;
1264                 }
1265                 /* clean ehca_mr_t, without changing struct ib_mr and lock */
1266                 save_mr = *e_mr;
1267                 ehca_mr_deletenew(e_mr);
1268
1269                 /* set some MR values */
1270                 e_mr->flags = save_mr.flags;
1271                 e_mr->fmr_page_size = save_mr.fmr_page_size;
1272                 e_mr->fmr_max_pages = save_mr.fmr_max_pages;
1273                 e_mr->fmr_max_maps = save_mr.fmr_max_maps;
1274                 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
1275
1276                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
1277                                       e_pd, pginfo, lkey, rkey);
1278                 if (ret) {
1279                         u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
1280                         memcpy(&e_mr->flags, &(save_mr.flags),
1281                                sizeof(struct ehca_mr) - offset);
1282                         goto ehca_rereg_mr_exit0;
1283                 }
1284         }
1285
1286 ehca_rereg_mr_exit0:
1287         if (ret)
1288                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1289                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1290                          "num_kpages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
1291                          "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
1292                          acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
1293                          rereg_1_hcall, rereg_3_hcall);
1294         return ret;
1295 } /* end ehca_rereg_mr() */
1296
1297 /*----------------------------------------------------------------------*/
1298
1299 int ehca_unmap_one_fmr(struct ehca_shca *shca,
1300                        struct ehca_mr *e_fmr)
1301 {
1302         int ret = 0;
1303         u64 h_ret;
1304         struct ehca_pd *e_pd =
1305                 container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
1306         struct ehca_mr save_fmr;
1307         u32 tmp_lkey, tmp_rkey;
1308         struct ehca_mr_pginfo pginfo;
1309         struct ehca_mr_hipzout_parms hipzout;
1310         struct ehca_mr save_mr;
1311
1312         if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
1313                 /*
1314                  * note: after using rereg hcall with len=0,
1315                  * rereg hcall must be used again for registering pages
1316                  */
1317                 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
1318                                               0, 0, e_pd->fw_pd, 0, &hipzout);
1319                 if (h_ret == H_SUCCESS) {
1320                         /* successful reregistration */
1321                         e_fmr->start = NULL;
1322                         e_fmr->size = 0;
1323                         tmp_lkey = hipzout.lkey;
1324                         tmp_rkey = hipzout.rkey;
1325                         return 0;
1326                 }
1327                 /*
1328                  * should not happen, because length checked above,
1329                  * FMRs are not shared and no MW bound to FMRs
1330                  */
1331                 ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
1332                          "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
1333                          "mr_hndl=%lx lkey=%x lkey_out=%x",
1334                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
1335                          e_fmr->ipz_mr_handle.handle,
1336                          e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
1337                 /* try free and rereg */
1338         }
1339
1340         /* first free old FMR */
1341         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
1342         if (h_ret != H_SUCCESS) {
1343                 ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1344                          "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
1345                          "lkey=%x",
1346                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
1347                          e_fmr->ipz_mr_handle.handle,
1348                          e_fmr->ib.ib_fmr.lkey);
1349                 ret = ehca2ib_return_code(h_ret);
1350                 goto ehca_unmap_one_fmr_exit0;
1351         }
1352         /* clean ehca_mr_t, without changing lock */
1353         save_fmr = *e_fmr;
1354         ehca_mr_deletenew(e_fmr);
1355
1356         /* set some MR values */
1357         e_fmr->flags = save_fmr.flags;
1358         e_fmr->fmr_page_size = save_fmr.fmr_page_size;
1359         e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
1360         e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
1361         e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
1362         e_fmr->acl = save_fmr.acl;
1363
1364         memset(&pginfo, 0, sizeof(pginfo));
1365         pginfo.type = EHCA_MR_PGI_FMR;
1366         pginfo.num_kpages = 0;
1367         pginfo.num_hwpages = 0;
1368         ret = ehca_reg_mr(shca, e_fmr, NULL,
1369                           (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
1370                           e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
1371                           &tmp_rkey);
1372         if (ret) {
1373                 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
1374                 memcpy(&e_fmr->flags, &(save_mr.flags),
1375                        sizeof(struct ehca_mr) - offset);
1376                 goto ehca_unmap_one_fmr_exit0;
1377         }
1378
1379 ehca_unmap_one_fmr_exit0:
1380         if (ret)
1381                 ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x "
1382                          "fmr_max_pages=%x",
1383                          ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
1384         return ret;
1385 } /* end ehca_unmap_one_fmr() */
1386
1387 /*----------------------------------------------------------------------*/
1388
1389 int ehca_reg_smr(struct ehca_shca *shca,
1390                  struct ehca_mr *e_origmr,
1391                  struct ehca_mr *e_newmr,
1392                  u64 *iova_start,
1393                  int acl,
1394                  struct ehca_pd *e_pd,
1395                  u32 *lkey, /*OUT*/
1396                  u32 *rkey) /*OUT*/
1397 {
1398         int ret = 0;
1399         u64 h_ret;
1400         u32 hipz_acl;
1401         struct ehca_mr_hipzout_parms hipzout;
1402
1403         ehca_mrmw_map_acl(acl, &hipz_acl);
1404         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1405
1406         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1407                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1408                                     &hipzout);
1409         if (h_ret != H_SUCCESS) {
1410                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1411                          "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
1412                          "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1413                          h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
1414                          shca->ipz_hca_handle.handle,
1415                          e_origmr->ipz_mr_handle.handle,
1416                          e_origmr->ib.ib_mr.lkey);
1417                 ret = ehca2ib_return_code(h_ret);
1418                 goto ehca_reg_smr_exit0;
1419         }
1420         /* successful registration */
1421         e_newmr->num_kpages = e_origmr->num_kpages;
1422         e_newmr->num_hwpages = e_origmr->num_hwpages;
1423         e_newmr->start = iova_start;
1424         e_newmr->size = e_origmr->size;
1425         e_newmr->acl = acl;
1426         e_newmr->ipz_mr_handle = hipzout.handle;
1427         *lkey = hipzout.lkey;
1428         *rkey = hipzout.rkey;
1429         return 0;
1430
1431 ehca_reg_smr_exit0:
1432         if (ret)
1433                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p "
1434                          "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
1435                          ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
1436         return ret;
1437 } /* end ehca_reg_smr() */
1438
1439 /*----------------------------------------------------------------------*/
1440
1441 /* register internal max-MR to internal SHCA */
1442 int ehca_reg_internal_maxmr(
1443         struct ehca_shca *shca,
1444         struct ehca_pd *e_pd,
1445         struct ehca_mr **e_maxmr)  /*OUT*/
1446 {
1447         int ret;
1448         struct ehca_mr *e_mr;
1449         u64 *iova_start;
1450         u64 size_maxmr;
1451         struct ehca_mr_pginfo pginfo;
1452         struct ib_phys_buf ib_pbuf;
1453         u32 num_kpages;
1454         u32 num_hwpages;
1455
1456         e_mr = ehca_mr_new();
1457         if (!e_mr) {
1458                 ehca_err(&shca->ib_device, "out of memory");
1459                 ret = -ENOMEM;
1460                 goto ehca_reg_internal_maxmr_exit0;
1461         }
1462         e_mr->flags |= EHCA_MR_FLAG_MAXMR;
1463
1464         /* register internal max-MR on HCA */
1465         size_maxmr = (u64)high_memory - PAGE_OFFSET;
1466         iova_start = (u64 *)KERNELBASE;
1467         ib_pbuf.addr = 0;
1468         ib_pbuf.size = size_maxmr;
1469         num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
1470                                 PAGE_SIZE);
1471         num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr,
1472                                  EHCA_PAGESIZE);
1473
1474         memset(&pginfo, 0, sizeof(pginfo));
1475         pginfo.type = EHCA_MR_PGI_PHYS;
1476         pginfo.num_kpages = num_kpages;
1477         pginfo.num_hwpages = num_hwpages;
1478         pginfo.u.phy.num_phys_buf = 1;
1479         pginfo.u.phy.phys_buf_array = &ib_pbuf;
1480
1481         ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
1482                           &pginfo, &e_mr->ib.ib_mr.lkey,
1483                           &e_mr->ib.ib_mr.rkey);
1484         if (ret) {
1485                 ehca_err(&shca->ib_device, "reg of internal max MR failed, "
1486                          "e_mr=%p iova_start=%p size_maxmr=%lx num_kpages=%x "
1487                          "num_hwpages=%x", e_mr, iova_start, size_maxmr,
1488                          num_kpages, num_hwpages);
1489                 goto ehca_reg_internal_maxmr_exit1;
1490         }
1491
1492         /* successful registration of all pages */
1493         e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
1494         e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
1495         e_mr->ib.ib_mr.uobject = NULL;
1496         atomic_inc(&(e_pd->ib_pd.usecnt));
1497         atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
1498         *e_maxmr = e_mr;
1499         return 0;
1500
1501 ehca_reg_internal_maxmr_exit1:
1502         ehca_mr_delete(e_mr);
1503 ehca_reg_internal_maxmr_exit0:
1504         if (ret)
1505                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
1506                          ret, shca, e_pd, e_maxmr);
1507         return ret;
1508 } /* end ehca_reg_internal_maxmr() */
1509
1510 /*----------------------------------------------------------------------*/
1511
1512 int ehca_reg_maxmr(struct ehca_shca *shca,
1513                    struct ehca_mr *e_newmr,
1514                    u64 *iova_start,
1515                    int acl,
1516                    struct ehca_pd *e_pd,
1517                    u32 *lkey,
1518                    u32 *rkey)
1519 {
1520         u64 h_ret;
1521         struct ehca_mr *e_origmr = shca->maxmr;
1522         u32 hipz_acl;
1523         struct ehca_mr_hipzout_parms hipzout;
1524
1525         ehca_mrmw_map_acl(acl, &hipz_acl);
1526         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1527
1528         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1529                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1530                                     &hipzout);
1531         if (h_ret != H_SUCCESS) {
1532                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1533                          "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1534                          h_ret, e_origmr, shca->ipz_hca_handle.handle,
1535                          e_origmr->ipz_mr_handle.handle,
1536                          e_origmr->ib.ib_mr.lkey);
1537                 return ehca2ib_return_code(h_ret);
1538         }
1539         /* successful registration */
1540         e_newmr->num_kpages = e_origmr->num_kpages;
1541         e_newmr->num_hwpages = e_origmr->num_hwpages;
1542         e_newmr->start = iova_start;
1543         e_newmr->size = e_origmr->size;
1544         e_newmr->acl = acl;
1545         e_newmr->ipz_mr_handle = hipzout.handle;
1546         *lkey = hipzout.lkey;
1547         *rkey = hipzout.rkey;
1548         return 0;
1549 } /* end ehca_reg_maxmr() */
1550
1551 /*----------------------------------------------------------------------*/
1552
1553 int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
1554 {
1555         int ret;
1556         struct ehca_mr *e_maxmr;
1557         struct ib_pd *ib_pd;
1558
1559         if (!shca->maxmr) {
1560                 ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
1561                 ret = -EINVAL;
1562                 goto ehca_dereg_internal_maxmr_exit0;
1563         }
1564
1565         e_maxmr = shca->maxmr;
1566         ib_pd = e_maxmr->ib.ib_mr.pd;
1567         shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
1568
1569         ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
1570         if (ret) {
1571                 ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
1572                          "ret=%x e_maxmr=%p shca=%p lkey=%x",
1573                          ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
1574                 shca->maxmr = e_maxmr;
1575                 goto ehca_dereg_internal_maxmr_exit0;
1576         }
1577
1578         atomic_dec(&ib_pd->usecnt);
1579
1580 ehca_dereg_internal_maxmr_exit0:
1581         if (ret)
1582                 ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p",
1583                          ret, shca, shca->maxmr);
1584         return ret;
1585 } /* end ehca_dereg_internal_maxmr() */
1586
1587 /*----------------------------------------------------------------------*/
1588
1589 /*
1590  * check physical buffer array of MR verbs for validness and
1591  * calculates MR size
1592  */
1593 int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
1594                                   int num_phys_buf,
1595                                   u64 *iova_start,
1596                                   u64 *size)
1597 {
1598         struct ib_phys_buf *pbuf = phys_buf_array;
1599         u64 size_count = 0;
1600         u32 i;
1601
1602         if (num_phys_buf == 0) {
1603                 ehca_gen_err("bad phys buf array len, num_phys_buf=0");
1604                 return -EINVAL;
1605         }
1606         /* check first buffer */
1607         if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
1608                 ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
1609                              "pbuf->addr=%lx pbuf->size=%lx",
1610                              iova_start, pbuf->addr, pbuf->size);
1611                 return -EINVAL;
1612         }
1613         if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
1614             (num_phys_buf > 1)) {
1615                 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
1616                              "pbuf->size=%lx", pbuf->addr, pbuf->size);
1617                 return -EINVAL;
1618         }
1619
1620         for (i = 0; i < num_phys_buf; i++) {
1621                 if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
1622                         ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
1623                                      "pbuf->size=%lx",
1624                                      i, pbuf->addr, pbuf->size);
1625                         return -EINVAL;
1626                 }
1627                 if (((i > 0) && /* not 1st */
1628                      (i < (num_phys_buf - 1)) &&        /* not last */
1629                      (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
1630                         ehca_gen_err("bad size, i=%x pbuf->size=%lx",
1631                                      i, pbuf->size);
1632                         return -EINVAL;
1633                 }
1634                 size_count += pbuf->size;
1635                 pbuf++;
1636         }
1637
1638         *size = size_count;
1639         return 0;
1640 } /* end ehca_mr_chk_buf_and_calc_size() */
1641
1642 /*----------------------------------------------------------------------*/
1643
1644 /* check page list of map FMR verb for validness */
1645 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
1646                              u64 *page_list,
1647                              int list_len)
1648 {
1649         u32 i;
1650         u64 *page;
1651
1652         if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
1653                 ehca_gen_err("bad list_len, list_len=%x "
1654                              "e_fmr->fmr_max_pages=%x fmr=%p",
1655                              list_len, e_fmr->fmr_max_pages, e_fmr);
1656                 return -EINVAL;
1657         }
1658
1659         /* each page must be aligned */
1660         page = page_list;
1661         for (i = 0; i < list_len; i++) {
1662                 if (*page % e_fmr->fmr_page_size) {
1663                         ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
1664                                      "fmr_page_size=%x", i, *page, page, e_fmr,
1665                                      e_fmr->fmr_page_size);
1666                         return -EINVAL;
1667                 }
1668                 page++;
1669         }
1670
1671         return 0;
1672 } /* end ehca_fmr_check_page_list() */
1673
1674 /*----------------------------------------------------------------------*/
1675
1676 /* PAGE_SIZE >= pginfo->hwpage_size */
1677 static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1678                                   u32 number,
1679                                   u64 *kpage)
1680 {
1681         int ret = 0;
1682         struct ib_umem_chunk *prev_chunk;
1683         struct ib_umem_chunk *chunk;
1684         u64 pgaddr;
1685         u32 i = 0;
1686         u32 j = 0;
1687
1688         /* loop over desired chunk entries */
1689         chunk      = pginfo->u.usr.next_chunk;
1690         prev_chunk = pginfo->u.usr.next_chunk;
1691         list_for_each_entry_continue(
1692                 chunk, (&(pginfo->u.usr.region->chunk_list)), list) {
1693                 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) {
1694                         pgaddr = page_to_pfn(chunk->page_list[i].page)
1695                                 << PAGE_SHIFT ;
1696                         *kpage = phys_to_abs(pgaddr +
1697                                              (pginfo->next_hwpage *
1698                                               EHCA_PAGESIZE));
1699                         if ( !(*kpage) ) {
1700                                 ehca_gen_err("pgaddr=%lx "
1701                                              "chunk->page_list[i]=%lx "
1702                                              "i=%x next_hwpage=%lx",
1703                                              pgaddr, (u64)sg_dma_address(
1704                                                      &chunk->page_list[i]),
1705                                              i, pginfo->next_hwpage);
1706                                 return -EFAULT;
1707                         }
1708                         (pginfo->hwpage_cnt)++;
1709                         (pginfo->next_hwpage)++;
1710                         kpage++;
1711                         if (pginfo->next_hwpage %
1712                             (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1713                                 (pginfo->kpage_cnt)++;
1714                                 (pginfo->u.usr.next_nmap)++;
1715                                 pginfo->next_hwpage = 0;
1716                                 i++;
1717                         }
1718                         j++;
1719                         if (j >= number) break;
1720                 }
1721                 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
1722                     (j >= number)) {
1723                         pginfo->u.usr.next_nmap = 0;
1724                         prev_chunk = chunk;
1725                         break;
1726                 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
1727                         pginfo->u.usr.next_nmap = 0;
1728                         prev_chunk = chunk;
1729                 } else if (j >= number)
1730                         break;
1731                 else
1732                         prev_chunk = chunk;
1733         }
1734         pginfo->u.usr.next_chunk =
1735                 list_prepare_entry(prev_chunk,
1736                                    (&(pginfo->u.usr.region->chunk_list)),
1737                                    list);
1738         return ret;
1739 }
1740
1741 int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1742                           u32 number,
1743                           u64 *kpage)
1744 {
1745         int ret = 0;
1746         struct ib_phys_buf *pbuf;
1747         u64 num_hw, offs_hw;
1748         u32 i = 0;
1749
1750         /* loop over desired phys_buf_array entries */
1751         while (i < number) {
1752                 pbuf   = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
1753                 num_hw  = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) +
1754                                      pbuf->size, EHCA_PAGESIZE);
1755                 offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
1756                 while (pginfo->next_hwpage < offs_hw + num_hw) {
1757                         /* sanity check */
1758                         if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
1759                             (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
1760                                 ehca_gen_err("kpage_cnt >= num_kpages, "
1761                                              "kpage_cnt=%lx num_kpages=%lx "
1762                                              "hwpage_cnt=%lx "
1763                                              "num_hwpages=%lx i=%x",
1764                                              pginfo->kpage_cnt,
1765                                              pginfo->num_kpages,
1766                                              pginfo->hwpage_cnt,
1767                                              pginfo->num_hwpages, i);
1768                                 return -EFAULT;
1769                         }
1770                         *kpage = phys_to_abs(
1771                                 (pbuf->addr & EHCA_PAGEMASK)
1772                                 + (pginfo->next_hwpage * EHCA_PAGESIZE));
1773                         if ( !(*kpage) && pbuf->addr ) {
1774                                 ehca_gen_err("pbuf->addr=%lx "
1775                                              "pbuf->size=%lx "
1776                                              "next_hwpage=%lx", pbuf->addr,
1777                                              pbuf->size,
1778                                              pginfo->next_hwpage);
1779                                 return -EFAULT;
1780                         }
1781                         (pginfo->hwpage_cnt)++;
1782                         (pginfo->next_hwpage)++;
1783                         if (pginfo->next_hwpage %
1784                             (PAGE_SIZE / EHCA_PAGESIZE) == 0)
1785                                 (pginfo->kpage_cnt)++;
1786                         kpage++;
1787                         i++;
1788                         if (i >= number) break;
1789                 }
1790                 if (pginfo->next_hwpage >= offs_hw + num_hw) {
1791                         (pginfo->u.phy.next_buf)++;
1792                         pginfo->next_hwpage = 0;
1793                 }
1794         }
1795         return ret;
1796 }
1797
1798 int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
1799                          u32 number,
1800                          u64 *kpage)
1801 {
1802         int ret = 0;
1803         u64 *fmrlist;
1804         u32 i;
1805
1806         /* loop over desired page_list entries */
1807         fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
1808         for (i = 0; i < number; i++) {
1809                 *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
1810                                      pginfo->next_hwpage * EHCA_PAGESIZE);
1811                 if ( !(*kpage) ) {
1812                         ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1813                                      "next_listelem=%lx next_hwpage=%lx",
1814                                      *fmrlist, fmrlist,
1815                                      pginfo->u.fmr.next_listelem,
1816                                      pginfo->next_hwpage);
1817                         return -EFAULT;
1818                 }
1819                 (pginfo->hwpage_cnt)++;
1820                 (pginfo->next_hwpage)++;
1821                 kpage++;
1822                 if (pginfo->next_hwpage %
1823                     (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) {
1824                         (pginfo->kpage_cnt)++;
1825                         (pginfo->u.fmr.next_listelem)++;
1826                         fmrlist++;
1827                         pginfo->next_hwpage = 0;
1828                 }
1829         }
1830         return ret;
1831 }
1832
1833 /* setup page buffer from page info */
1834 int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
1835                      u32 number,
1836                      u64 *kpage)
1837 {
1838         int ret;
1839
1840         switch (pginfo->type) {
1841         case EHCA_MR_PGI_PHYS:
1842                 ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
1843                 break;
1844         case EHCA_MR_PGI_USER:
1845                 ret = ehca_set_pagebuf_user1(pginfo, number, kpage);
1846                 break;
1847         case EHCA_MR_PGI_FMR:
1848                 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
1849                 break;
1850         default:
1851                 ehca_gen_err("bad pginfo->type=%x", pginfo->type);
1852                 ret = -EFAULT;
1853                 break;
1854         }
1855         return ret;
1856 } /* end ehca_set_pagebuf() */
1857
1858 /*----------------------------------------------------------------------*/
1859
1860 /*
1861  * check MR if it is a max-MR, i.e. uses whole memory
1862  * in case it's a max-MR 1 is returned, else 0
1863  */
1864 int ehca_mr_is_maxmr(u64 size,
1865                      u64 *iova_start)
1866 {
1867         /* a MR is treated as max-MR only if it fits following: */
1868         if ((size == ((u64)high_memory - PAGE_OFFSET)) &&
1869             (iova_start == (void *)KERNELBASE)) {
1870                 ehca_gen_dbg("this is a max-MR");
1871                 return 1;
1872         } else
1873                 return 0;
1874 } /* end ehca_mr_is_maxmr() */
1875
1876 /*----------------------------------------------------------------------*/
1877
1878 /* map access control for MR/MW. This routine is used for MR and MW. */
1879 void ehca_mrmw_map_acl(int ib_acl,
1880                        u32 *hipz_acl)
1881 {
1882         *hipz_acl = 0;
1883         if (ib_acl & IB_ACCESS_REMOTE_READ)
1884                 *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
1885         if (ib_acl & IB_ACCESS_REMOTE_WRITE)
1886                 *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
1887         if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
1888                 *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
1889         if (ib_acl & IB_ACCESS_LOCAL_WRITE)
1890                 *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
1891         if (ib_acl & IB_ACCESS_MW_BIND)
1892                 *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
1893 } /* end ehca_mrmw_map_acl() */
1894
1895 /*----------------------------------------------------------------------*/
1896
1897 /* sets page size in hipz access control for MR/MW. */
1898 void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
1899 {
1900         return; /* HCA supports only 4k */
1901 } /* end ehca_mrmw_set_pgsize_hipz_acl() */
1902
1903 /*----------------------------------------------------------------------*/
1904
1905 /*
1906  * reverse map access control for MR/MW.
1907  * This routine is used for MR and MW.
1908  */
1909 void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
1910                                int *ib_acl) /*OUT*/
1911 {
1912         *ib_acl = 0;
1913         if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
1914                 *ib_acl |= IB_ACCESS_REMOTE_READ;
1915         if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
1916                 *ib_acl |= IB_ACCESS_REMOTE_WRITE;
1917         if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
1918                 *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
1919         if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
1920                 *ib_acl |= IB_ACCESS_LOCAL_WRITE;
1921         if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
1922                 *ib_acl |= IB_ACCESS_MW_BIND;
1923 } /* end ehca_mrmw_reverse_map_acl() */
1924
1925
1926 /*----------------------------------------------------------------------*/
1927
1928 /*
1929  * MR destructor and constructor
1930  * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
1931  * except struct ib_mr and spinlock
1932  */
1933 void ehca_mr_deletenew(struct ehca_mr *mr)
1934 {
1935         mr->flags = 0;
1936         mr->num_kpages = 0;
1937         mr->num_hwpages = 0;
1938         mr->acl = 0;
1939         mr->start = NULL;
1940         mr->fmr_page_size = 0;
1941         mr->fmr_max_pages = 0;
1942         mr->fmr_max_maps = 0;
1943         mr->fmr_map_cnt = 0;
1944         memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
1945         memset(&mr->galpas, 0, sizeof(mr->galpas));
1946 } /* end ehca_mr_deletenew() */
1947
1948 int ehca_init_mrmw_cache(void)
1949 {
1950         mr_cache = kmem_cache_create("ehca_cache_mr",
1951                                      sizeof(struct ehca_mr), 0,
1952                                      SLAB_HWCACHE_ALIGN,
1953                                      NULL);
1954         if (!mr_cache)
1955                 return -ENOMEM;
1956         mw_cache = kmem_cache_create("ehca_cache_mw",
1957                                      sizeof(struct ehca_mw), 0,
1958                                      SLAB_HWCACHE_ALIGN,
1959                                      NULL);
1960         if (!mw_cache) {
1961                 kmem_cache_destroy(mr_cache);
1962                 mr_cache = NULL;
1963                 return -ENOMEM;
1964         }
1965         return 0;
1966 }
1967
1968 void ehca_cleanup_mrmw_cache(void)
1969 {
1970         if (mr_cache)
1971                 kmem_cache_destroy(mr_cache);
1972         if (mw_cache)
1973                 kmem_cache_destroy(mw_cache);
1974 }