0596ec16fcbd2bfd2b4f1b23bfa45d23b5f86156
[linux-2.6.git] / drivers / infiniband / hw / ipath / ipath_uc.c
1 /*
2  * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
3  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #include "ipath_verbs.h"
35 #include "ipath_kernel.h"
36
37 /* cut down ridiculously long IB macro names */
38 #define OP(x) IB_OPCODE_UC_##x
39
40 /**
41  * ipath_make_uc_req - construct a request packet (SEND, RDMA write)
42  * @qp: a pointer to the QP
43  *
44  * Return 1 if constructed; otherwise, return 0.
45  */
46 int ipath_make_uc_req(struct ipath_qp *qp)
47 {
48         struct ipath_other_headers *ohdr;
49         struct ipath_swqe *wqe;
50         unsigned long flags;
51         u32 hwords;
52         u32 bth0;
53         u32 len;
54         u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
55         int ret = 0;
56
57         spin_lock_irqsave(&qp->s_lock, flags);
58
59         if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {
60                 if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))
61                         goto bail;
62                 /* We are in the error state, flush the work request. */
63                 if (qp->s_last == qp->s_head)
64                         goto bail;
65                 /* If DMAs are in progress, we can't flush immediately. */
66                 if (atomic_read(&qp->s_dma_busy)) {
67                         qp->s_flags |= IPATH_S_WAIT_DMA;
68                         goto bail;
69                 }
70                 wqe = get_swqe_ptr(qp, qp->s_last);
71                 ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
72                 goto done;
73         }
74
75         ohdr = &qp->s_hdr.u.oth;
76         if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
77                 ohdr = &qp->s_hdr.u.l.oth;
78
79         /* header size in 32-bit words LRH+BTH = (8+12)/4. */
80         hwords = 5;
81         bth0 = 1 << 22; /* Set M bit */
82
83         /* Get the next send request. */
84         wqe = get_swqe_ptr(qp, qp->s_cur);
85         qp->s_wqe = NULL;
86         switch (qp->s_state) {
87         default:
88                 if (!(ib_ipath_state_ops[qp->state] &
89                     IPATH_PROCESS_NEXT_SEND_OK))
90                         goto bail;
91                 /* Check if send work queue is empty. */
92                 if (qp->s_cur == qp->s_head)
93                         goto bail;
94                 /*
95                  * Start a new request.
96                  */
97                 qp->s_psn = wqe->psn = qp->s_next_psn;
98                 qp->s_sge.sge = wqe->sg_list[0];
99                 qp->s_sge.sg_list = wqe->sg_list + 1;
100                 qp->s_sge.num_sge = wqe->wr.num_sge;
101                 qp->s_len = len = wqe->length;
102                 switch (wqe->wr.opcode) {
103                 case IB_WR_SEND:
104                 case IB_WR_SEND_WITH_IMM:
105                         if (len > pmtu) {
106                                 qp->s_state = OP(SEND_FIRST);
107                                 len = pmtu;
108                                 break;
109                         }
110                         if (wqe->wr.opcode == IB_WR_SEND)
111                                 qp->s_state = OP(SEND_ONLY);
112                         else {
113                                 qp->s_state =
114                                         OP(SEND_ONLY_WITH_IMMEDIATE);
115                                 /* Immediate data comes after the BTH */
116                                 ohdr->u.imm_data = wqe->wr.ex.imm_data;
117                                 hwords += 1;
118                         }
119                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
120                                 bth0 |= 1 << 23;
121                         qp->s_wqe = wqe;
122                         if (++qp->s_cur >= qp->s_size)
123                                 qp->s_cur = 0;
124                         break;
125
126                 case IB_WR_RDMA_WRITE:
127                 case IB_WR_RDMA_WRITE_WITH_IMM:
128                         ohdr->u.rc.reth.vaddr =
129                                 cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
130                         ohdr->u.rc.reth.rkey =
131                                 cpu_to_be32(wqe->wr.wr.rdma.rkey);
132                         ohdr->u.rc.reth.length = cpu_to_be32(len);
133                         hwords += sizeof(struct ib_reth) / 4;
134                         if (len > pmtu) {
135                                 qp->s_state = OP(RDMA_WRITE_FIRST);
136                                 len = pmtu;
137                                 break;
138                         }
139                         if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
140                                 qp->s_state = OP(RDMA_WRITE_ONLY);
141                         else {
142                                 qp->s_state =
143                                         OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
144                                 /* Immediate data comes after the RETH */
145                                 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
146                                 hwords += 1;
147                                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
148                                         bth0 |= 1 << 23;
149                         }
150                         qp->s_wqe = wqe;
151                         if (++qp->s_cur >= qp->s_size)
152                                 qp->s_cur = 0;
153                         break;
154
155                 default:
156                         goto bail;
157                 }
158                 break;
159
160         case OP(SEND_FIRST):
161                 qp->s_state = OP(SEND_MIDDLE);
162                 /* FALLTHROUGH */
163         case OP(SEND_MIDDLE):
164                 len = qp->s_len;
165                 if (len > pmtu) {
166                         len = pmtu;
167                         break;
168                 }
169                 if (wqe->wr.opcode == IB_WR_SEND)
170                         qp->s_state = OP(SEND_LAST);
171                 else {
172                         qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
173                         /* Immediate data comes after the BTH */
174                         ohdr->u.imm_data = wqe->wr.ex.imm_data;
175                         hwords += 1;
176                 }
177                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
178                         bth0 |= 1 << 23;
179                 qp->s_wqe = wqe;
180                 if (++qp->s_cur >= qp->s_size)
181                         qp->s_cur = 0;
182                 break;
183
184         case OP(RDMA_WRITE_FIRST):
185                 qp->s_state = OP(RDMA_WRITE_MIDDLE);
186                 /* FALLTHROUGH */
187         case OP(RDMA_WRITE_MIDDLE):
188                 len = qp->s_len;
189                 if (len > pmtu) {
190                         len = pmtu;
191                         break;
192                 }
193                 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
194                         qp->s_state = OP(RDMA_WRITE_LAST);
195                 else {
196                         qp->s_state =
197                                 OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
198                         /* Immediate data comes after the BTH */
199                         ohdr->u.imm_data = wqe->wr.ex.imm_data;
200                         hwords += 1;
201                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
202                                 bth0 |= 1 << 23;
203                 }
204                 qp->s_wqe = wqe;
205                 if (++qp->s_cur >= qp->s_size)
206                         qp->s_cur = 0;
207                 break;
208         }
209         qp->s_len -= len;
210         qp->s_hdrwords = hwords;
211         qp->s_cur_sge = &qp->s_sge;
212         qp->s_cur_size = len;
213         ipath_make_ruc_header(to_idev(qp->ibqp.device),
214                               qp, ohdr, bth0 | (qp->s_state << 24),
215                               qp->s_next_psn++ & IPATH_PSN_MASK);
216 done:
217         ret = 1;
218         goto unlock;
219
220 bail:
221         qp->s_flags &= ~IPATH_S_BUSY;
222 unlock:
223         spin_unlock_irqrestore(&qp->s_lock, flags);
224         return ret;
225 }
226
227 /**
228  * ipath_uc_rcv - handle an incoming UC packet
229  * @dev: the device the packet came in on
230  * @hdr: the header of the packet
231  * @has_grh: true if the packet has a GRH
232  * @data: the packet data
233  * @tlen: the length of the packet
234  * @qp: the QP for this packet.
235  *
236  * This is called from ipath_qp_rcv() to process an incoming UC packet
237  * for the given QP.
238  * Called at interrupt level.
239  */
240 void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
241                   int has_grh, void *data, u32 tlen, struct ipath_qp *qp)
242 {
243         struct ipath_other_headers *ohdr;
244         int opcode;
245         u32 hdrsize;
246         u32 psn;
247         u32 pad;
248         struct ib_wc wc;
249         u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
250         struct ib_reth *reth;
251         int header_in_data;
252
253         /* Validate the SLID. See Ch. 9.6.1.5 */
254         if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid))
255                 goto done;
256
257         /* Check for GRH */
258         if (!has_grh) {
259                 ohdr = &hdr->u.oth;
260                 hdrsize = 8 + 12;       /* LRH + BTH */
261                 psn = be32_to_cpu(ohdr->bth[2]);
262                 header_in_data = 0;
263         } else {
264                 ohdr = &hdr->u.l.oth;
265                 hdrsize = 8 + 40 + 12;  /* LRH + GRH + BTH */
266                 /*
267                  * The header with GRH is 60 bytes and the
268                  * core driver sets the eager header buffer
269                  * size to 56 bytes so the last 4 bytes of
270                  * the BTH header (PSN) is in the data buffer.
271                  */
272                 header_in_data = dev->dd->ipath_rcvhdrentsize == 16;
273                 if (header_in_data) {
274                         psn = be32_to_cpu(((__be32 *) data)[0]);
275                         data += sizeof(__be32);
276                 } else
277                         psn = be32_to_cpu(ohdr->bth[2]);
278         }
279         /*
280          * The opcode is in the low byte when its in network order
281          * (top byte when in host order).
282          */
283         opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
284
285         memset(&wc, 0, sizeof wc);
286
287         /* Compare the PSN verses the expected PSN. */
288         if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) {
289                 /*
290                  * Handle a sequence error.
291                  * Silently drop any current message.
292                  */
293                 qp->r_psn = psn;
294         inv:
295                 qp->r_state = OP(SEND_LAST);
296                 switch (opcode) {
297                 case OP(SEND_FIRST):
298                 case OP(SEND_ONLY):
299                 case OP(SEND_ONLY_WITH_IMMEDIATE):
300                         goto send_first;
301
302                 case OP(RDMA_WRITE_FIRST):
303                 case OP(RDMA_WRITE_ONLY):
304                 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
305                         goto rdma_first;
306
307                 default:
308                         dev->n_pkt_drops++;
309                         goto done;
310                 }
311         }
312
313         /* Check for opcode sequence errors. */
314         switch (qp->r_state) {
315         case OP(SEND_FIRST):
316         case OP(SEND_MIDDLE):
317                 if (opcode == OP(SEND_MIDDLE) ||
318                     opcode == OP(SEND_LAST) ||
319                     opcode == OP(SEND_LAST_WITH_IMMEDIATE))
320                         break;
321                 goto inv;
322
323         case OP(RDMA_WRITE_FIRST):
324         case OP(RDMA_WRITE_MIDDLE):
325                 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
326                     opcode == OP(RDMA_WRITE_LAST) ||
327                     opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
328                         break;
329                 goto inv;
330
331         default:
332                 if (opcode == OP(SEND_FIRST) ||
333                     opcode == OP(SEND_ONLY) ||
334                     opcode == OP(SEND_ONLY_WITH_IMMEDIATE) ||
335                     opcode == OP(RDMA_WRITE_FIRST) ||
336                     opcode == OP(RDMA_WRITE_ONLY) ||
337                     opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
338                         break;
339                 goto inv;
340         }
341
342         /* OK, process the packet. */
343         switch (opcode) {
344         case OP(SEND_FIRST):
345         case OP(SEND_ONLY):
346         case OP(SEND_ONLY_WITH_IMMEDIATE):
347         send_first:
348                 if (qp->r_flags & IPATH_R_REUSE_SGE) {
349                         qp->r_flags &= ~IPATH_R_REUSE_SGE;
350                         qp->r_sge = qp->s_rdma_read_sge;
351                 } else if (!ipath_get_rwqe(qp, 0)) {
352                         dev->n_pkt_drops++;
353                         goto done;
354                 }
355                 /* Save the WQE so we can reuse it in case of an error. */
356                 qp->s_rdma_read_sge = qp->r_sge;
357                 qp->r_rcv_len = 0;
358                 if (opcode == OP(SEND_ONLY))
359                         goto send_last;
360                 else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))
361                         goto send_last_imm;
362                 /* FALLTHROUGH */
363         case OP(SEND_MIDDLE):
364                 /* Check for invalid length PMTU or posted rwqe len. */
365                 if (unlikely(tlen != (hdrsize + pmtu + 4))) {
366                         qp->r_flags |= IPATH_R_REUSE_SGE;
367                         dev->n_pkt_drops++;
368                         goto done;
369                 }
370                 qp->r_rcv_len += pmtu;
371                 if (unlikely(qp->r_rcv_len > qp->r_len)) {
372                         qp->r_flags |= IPATH_R_REUSE_SGE;
373                         dev->n_pkt_drops++;
374                         goto done;
375                 }
376                 ipath_copy_sge(&qp->r_sge, data, pmtu);
377                 break;
378
379         case OP(SEND_LAST_WITH_IMMEDIATE):
380         send_last_imm:
381                 if (header_in_data) {
382                         wc.imm_data = *(__be32 *) data;
383                         data += sizeof(__be32);
384                 } else {
385                         /* Immediate data comes after BTH */
386                         wc.imm_data = ohdr->u.imm_data;
387                 }
388                 hdrsize += 4;
389                 wc.wc_flags = IB_WC_WITH_IMM;
390                 /* FALLTHROUGH */
391         case OP(SEND_LAST):
392         send_last:
393                 /* Get the number of bytes the message was padded by. */
394                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
395                 /* Check for invalid length. */
396                 /* XXX LAST len should be >= 1 */
397                 if (unlikely(tlen < (hdrsize + pad + 4))) {
398                         qp->r_flags |= IPATH_R_REUSE_SGE;
399                         dev->n_pkt_drops++;
400                         goto done;
401                 }
402                 /* Don't count the CRC. */
403                 tlen -= (hdrsize + pad + 4);
404                 wc.byte_len = tlen + qp->r_rcv_len;
405                 if (unlikely(wc.byte_len > qp->r_len)) {
406                         qp->r_flags |= IPATH_R_REUSE_SGE;
407                         dev->n_pkt_drops++;
408                         goto done;
409                 }
410                 wc.opcode = IB_WC_RECV;
411         last_imm:
412                 ipath_copy_sge(&qp->r_sge, data, tlen);
413                 wc.wr_id = qp->r_wr_id;
414                 wc.status = IB_WC_SUCCESS;
415                 wc.qp = &qp->ibqp;
416                 wc.src_qp = qp->remote_qpn;
417                 wc.slid = qp->remote_ah_attr.dlid;
418                 wc.sl = qp->remote_ah_attr.sl;
419                 /* Signal completion event if the solicited bit is set. */
420                 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
421                                (ohdr->bth[0] &
422                                 __constant_cpu_to_be32(1 << 23)) != 0);
423                 break;
424
425         case OP(RDMA_WRITE_FIRST):
426         case OP(RDMA_WRITE_ONLY):
427         case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */
428         rdma_first:
429                 /* RETH comes after BTH */
430                 if (!header_in_data)
431                         reth = &ohdr->u.rc.reth;
432                 else {
433                         reth = (struct ib_reth *)data;
434                         data += sizeof(*reth);
435                 }
436                 hdrsize += sizeof(*reth);
437                 qp->r_len = be32_to_cpu(reth->length);
438                 qp->r_rcv_len = 0;
439                 if (qp->r_len != 0) {
440                         u32 rkey = be32_to_cpu(reth->rkey);
441                         u64 vaddr = be64_to_cpu(reth->vaddr);
442                         int ok;
443
444                         /* Check rkey */
445                         ok = ipath_rkey_ok(qp, &qp->r_sge, qp->r_len,
446                                            vaddr, rkey,
447                                            IB_ACCESS_REMOTE_WRITE);
448                         if (unlikely(!ok)) {
449                                 dev->n_pkt_drops++;
450                                 goto done;
451                         }
452                 } else {
453                         qp->r_sge.sg_list = NULL;
454                         qp->r_sge.sge.mr = NULL;
455                         qp->r_sge.sge.vaddr = NULL;
456                         qp->r_sge.sge.length = 0;
457                         qp->r_sge.sge.sge_length = 0;
458                 }
459                 if (unlikely(!(qp->qp_access_flags &
460                                IB_ACCESS_REMOTE_WRITE))) {
461                         dev->n_pkt_drops++;
462                         goto done;
463                 }
464                 if (opcode == OP(RDMA_WRITE_ONLY))
465                         goto rdma_last;
466                 else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
467                         goto rdma_last_imm;
468                 /* FALLTHROUGH */
469         case OP(RDMA_WRITE_MIDDLE):
470                 /* Check for invalid length PMTU or posted rwqe len. */
471                 if (unlikely(tlen != (hdrsize + pmtu + 4))) {
472                         dev->n_pkt_drops++;
473                         goto done;
474                 }
475                 qp->r_rcv_len += pmtu;
476                 if (unlikely(qp->r_rcv_len > qp->r_len)) {
477                         dev->n_pkt_drops++;
478                         goto done;
479                 }
480                 ipath_copy_sge(&qp->r_sge, data, pmtu);
481                 break;
482
483         case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
484         rdma_last_imm:
485                 if (header_in_data) {
486                         wc.imm_data = *(__be32 *) data;
487                         data += sizeof(__be32);
488                 } else {
489                         /* Immediate data comes after BTH */
490                         wc.imm_data = ohdr->u.imm_data;
491                 }
492                 hdrsize += 4;
493                 wc.wc_flags = IB_WC_WITH_IMM;
494
495                 /* Get the number of bytes the message was padded by. */
496                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
497                 /* Check for invalid length. */
498                 /* XXX LAST len should be >= 1 */
499                 if (unlikely(tlen < (hdrsize + pad + 4))) {
500                         dev->n_pkt_drops++;
501                         goto done;
502                 }
503                 /* Don't count the CRC. */
504                 tlen -= (hdrsize + pad + 4);
505                 if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) {
506                         dev->n_pkt_drops++;
507                         goto done;
508                 }
509                 if (qp->r_flags & IPATH_R_REUSE_SGE)
510                         qp->r_flags &= ~IPATH_R_REUSE_SGE;
511                 else if (!ipath_get_rwqe(qp, 1)) {
512                         dev->n_pkt_drops++;
513                         goto done;
514                 }
515                 wc.byte_len = qp->r_len;
516                 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
517                 goto last_imm;
518
519         case OP(RDMA_WRITE_LAST):
520         rdma_last:
521                 /* Get the number of bytes the message was padded by. */
522                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
523                 /* Check for invalid length. */
524                 /* XXX LAST len should be >= 1 */
525                 if (unlikely(tlen < (hdrsize + pad + 4))) {
526                         dev->n_pkt_drops++;
527                         goto done;
528                 }
529                 /* Don't count the CRC. */
530                 tlen -= (hdrsize + pad + 4);
531                 if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) {
532                         dev->n_pkt_drops++;
533                         goto done;
534                 }
535                 ipath_copy_sge(&qp->r_sge, data, tlen);
536                 break;
537
538         default:
539                 /* Drop packet for unknown opcodes. */
540                 dev->n_pkt_drops++;
541                 goto done;
542         }
543         qp->r_psn++;
544         qp->r_state = opcode;
545 done:
546         return;
547 }