[SCSI] open-iscsi/linux-iscsi-5 Initiator: Initiator code
[linux-2.6.git] / drivers / scsi / iscsi_tcp.c
1 /*
2  * iSCSI Initiator over TCP/IP Data-Path
3  *
4  * Copyright (C) 2004 Dmitry Yusupov
5  * Copyright (C) 2004 Alex Aizman
6  * Copyright (C) 2005 Mike Christie
7  * maintained by open-iscsi@googlegroups.com
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published
11  * by the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful, but
15  * WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17  * General Public License for more details.
18  *
19  * See the file COPYING included with this distribution for more details.
20  *
21  * Credits:
22  *      Christoph Hellwig
23  *      FUJITA Tomonori
24  *      Arne Redlich
25  *      Zhenyu Wang
26  */
27
28 #include <linux/types.h>
29 #include <linux/list.h>
30 #include <linux/inet.h>
31 #include <linux/blkdev.h>
32 #include <linux/crypto.h>
33 #include <linux/delay.h>
34 #include <linux/kfifo.h>
35 #include <linux/scatterlist.h>
36 #include <net/tcp.h>
37 #include <scsi/scsi_cmnd.h>
38 #include <scsi/scsi_device.h>
39 #include <scsi/scsi_eh.h>
40 #include <scsi/scsi_request.h>
41 #include <scsi/scsi_tcq.h>
42 #include <scsi/scsi_host.h>
43 #include <scsi/scsi.h>
44 #include <scsi/scsi_transport_iscsi.h>
45
46 #include "iscsi_tcp.h"
47
48 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
49               "Alex Aizman <itn780@yahoo.com>");
50 MODULE_DESCRIPTION("iSCSI/TCP data-path");
51 MODULE_LICENSE("GPL");
52
53 /* #define DEBUG_TCP */
54 /* #define DEBUG_SCSI */
55 #define DEBUG_ASSERT
56
57 #ifdef DEBUG_TCP
58 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
59 #else
60 #define debug_tcp(fmt...)
61 #endif
62
63 #ifdef DEBUG_SCSI
64 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
65 #else
66 #define debug_scsi(fmt...)
67 #endif
68
69 #ifndef DEBUG_ASSERT
70 #ifdef BUG_ON
71 #undef BUG_ON
72 #endif
73 #define BUG_ON(expr)
74 #endif
75
76 #define INVALID_SN_DELTA        0xffff
77
78 static unsigned int iscsi_max_lun = 512;
79 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
80
81 /* global data */
82 static kmem_cache_t *taskcache;
83
84 static inline void
85 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
86 {
87         sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
88         ibuf->sent = 0;
89 }
90
91 static inline void
92 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
93 {
94         ibuf->sg.page = (void*)vbuf;
95         ibuf->sg.offset = (unsigned int)-1;
96         ibuf->sg.length = size;
97         ibuf->sent = 0;
98 }
99
100 static inline void*
101 iscsi_buf_iov_base(struct iscsi_buf *ibuf)
102 {
103         return (char*)ibuf->sg.page + ibuf->sent;
104 }
105
106 static inline void
107 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
108 {
109         /*
110          * Fastpath: sg element fits into single page
111          */
112         if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) {
113                 ibuf->sg.page = sg->page;
114                 ibuf->sg.offset = sg->offset;
115                 ibuf->sg.length = sg->length;
116         } else
117                 iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length);
118         ibuf->sent = 0;
119 }
120
121 static inline int
122 iscsi_buf_left(struct iscsi_buf *ibuf)
123 {
124         int rc;
125
126         rc = ibuf->sg.length - ibuf->sent;
127         BUG_ON(rc < 0);
128         return rc;
129 }
130
131 static inline void
132 iscsi_buf_init_hdr(struct iscsi_conn *conn, struct iscsi_buf *ibuf,
133                    char *vbuf, u8 *crc)
134 {
135         iscsi_buf_init_virt(ibuf, vbuf, sizeof(struct iscsi_hdr));
136         if (conn->hdrdgst_en) {
137                 crypto_digest_digest(conn->tx_tfm, &ibuf->sg, 1, crc);
138                 ibuf->sg.length += sizeof(uint32_t);
139         }
140 }
141
142 static void
143 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
144 {
145         struct iscsi_session *session = conn->session;
146         unsigned long flags;
147
148         spin_lock_irqsave(&session->lock, flags);
149         if (session->conn_cnt == 1 || session->leadconn == conn)
150                 session->state = ISCSI_STATE_FAILED;
151         spin_unlock_irqrestore(&session->lock, flags);
152         set_bit(SUSPEND_BIT, &conn->suspend_tx);
153         set_bit(SUSPEND_BIT, &conn->suspend_rx);
154         iscsi_conn_error(iscsi_handle(conn), err);
155 }
156
157 static inline int
158 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
159 {
160         uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
161         uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
162
163         if (max_cmdsn < exp_cmdsn -1 &&
164             max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
165                 return ISCSI_ERR_MAX_CMDSN;
166         if (max_cmdsn > session->max_cmdsn ||
167             max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
168                 session->max_cmdsn = max_cmdsn;
169         if (exp_cmdsn > session->exp_cmdsn ||
170             exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
171                 session->exp_cmdsn = exp_cmdsn;
172
173         return 0;
174 }
175
176 static inline int
177 iscsi_hdr_extract(struct iscsi_conn *conn)
178 {
179         struct sk_buff *skb = conn->in.skb;
180
181         if (conn->in.copy >= conn->hdr_size &&
182             conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
183                 /*
184                  * Zero-copy PDU Header: using connection context
185                  * to store header pointer.
186                  */
187                 if (skb_shinfo(skb)->frag_list == NULL &&
188                     !skb_shinfo(skb)->nr_frags)
189                         conn->in.hdr = (struct iscsi_hdr *)
190                                 ((char*)skb->data + conn->in.offset);
191                 else {
192                         /* ignoring return code since we checked
193                          * in.copy before */
194                         skb_copy_bits(skb, conn->in.offset,
195                                 &conn->hdr, conn->hdr_size);
196                         conn->in.hdr = &conn->hdr;
197                 }
198                 conn->in.offset += conn->hdr_size;
199                 conn->in.copy -= conn->hdr_size;
200         } else {
201                 int hdr_remains;
202                 int copylen;
203
204                 /*
205                  * PDU header scattered across SKB's,
206                  * copying it... This'll happen quite rarely.
207                  */
208
209                 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
210                         conn->in.hdr_offset = 0;
211
212                 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
213                 BUG_ON(hdr_remains <= 0);
214
215                 copylen = min(conn->in.copy, hdr_remains);
216                 skb_copy_bits(skb, conn->in.offset,
217                         (char*)&conn->hdr + conn->in.hdr_offset, copylen);
218
219                 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
220                        "in.copy %d\n", conn->in.hdr_offset, copylen,
221                        conn->in.offset, conn->in.copy);
222
223                 conn->in.offset += copylen;
224                 conn->in.copy -= copylen;
225                 if (copylen < hdr_remains)  {
226                         conn->in_progress = IN_PROGRESS_HEADER_GATHER;
227                         conn->in.hdr_offset += copylen;
228                         return -EAGAIN;
229                 }
230                 conn->in.hdr = &conn->hdr;
231                 conn->discontiguous_hdr_cnt++;
232                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
233         }
234
235         return 0;
236 }
237
238 static inline void
239 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
240 {
241         struct scsi_cmnd *sc = ctask->sc;
242         struct iscsi_session *session = conn->session;
243
244         spin_lock(&session->lock);
245         if (unlikely(!sc)) {
246                 spin_unlock(&session->lock);
247                 return;
248         }
249         if (sc->sc_data_direction == DMA_TO_DEVICE) {
250                 struct iscsi_data_task *dtask, *n;
251                 /* WRITE: cleanup Data-Out's if any */
252                 spin_lock(&conn->lock);
253                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
254                         list_del(&dtask->item);
255                         mempool_free(dtask, ctask->datapool);
256                 }
257                 spin_unlock(&conn->lock);
258         }
259         ctask->xmstate = XMSTATE_IDLE;
260         ctask->r2t = NULL;
261         ctask->sc = NULL;
262         __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
263         spin_unlock(&session->lock);
264 }
265
266 /**
267  * iscsi_cmd_rsp - SCSI Command Response processing
268  * @conn: iscsi connection
269  * @ctask: scsi command task
270  **/
271 static int
272 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
273 {
274         int rc;
275         struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
276         struct iscsi_session *session = conn->session;
277         struct scsi_cmnd *sc = ctask->sc;
278
279         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
280         if (rc) {
281                 sc->result = (DID_ERROR << 16);
282                 goto out;
283         }
284
285         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
286
287         sc->result = (DID_OK << 16) | rhdr->cmd_status;
288
289         if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
290                 sc->result = (DID_ERROR << 16);
291                 goto out;
292         }
293
294         if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
295                 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
296
297                 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
298                 debug_scsi("copied %d bytes of sense\n", sensecopy);
299         }
300
301         if (sc->sc_data_direction == DMA_TO_DEVICE)
302                 goto out;
303
304         if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
305                 int res_count = be32_to_cpu(rhdr->residual_count);
306
307                 if (res_count > 0 && res_count <= sc->request_bufflen)
308                         sc->resid = res_count;
309                 else
310                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
311         } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
312                 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
313         else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
314                 sc->resid = be32_to_cpu(rhdr->residual_count);
315
316 out:
317         debug_scsi("done [sc %lx res %d itt 0x%x]\n",
318                    (long)sc, sc->result, ctask->itt);
319         conn->scsirsp_pdus_cnt++;
320         iscsi_ctask_cleanup(conn, ctask);
321         sc->scsi_done(sc);
322         return rc;
323 }
324
325 /**
326  * iscsi_data_rsp - SCSI Data-In Response processing
327  * @conn: iscsi connection
328  * @ctask: scsi command task
329  **/
330 static int
331 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
332 {
333         int rc;
334         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
335         struct iscsi_session *session = conn->session;
336         int datasn = be32_to_cpu(rhdr->datasn);
337
338         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
339         if (rc)
340                 return rc;
341         /*
342          * setup Data-In byte counter (gets decremented..)
343          */
344         ctask->data_count = conn->in.datalen;
345
346         if (conn->in.datalen == 0)
347                 return 0;
348
349         if (ctask->datasn != datasn)
350                 return ISCSI_ERR_DATASN;
351
352         ctask->datasn++;
353
354         ctask->data_offset = be32_to_cpu(rhdr->offset);
355         if (ctask->data_offset + conn->in.datalen > ctask->total_length)
356                 return ISCSI_ERR_DATA_OFFSET;
357
358         if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
359                 struct scsi_cmnd *sc = ctask->sc;
360
361                 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
362                 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
363                         int res_count = be32_to_cpu(rhdr->residual_count);
364
365                         if (res_count > 0 &&
366                             res_count <= sc->request_bufflen) {
367                                 sc->resid = res_count;
368                                 sc->result = (DID_OK << 16) | rhdr->cmd_status;
369                         } else
370                                 sc->result = (DID_BAD_TARGET << 16) |
371                                         rhdr->cmd_status;
372                 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
373                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
374                 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) {
375                         sc->resid = be32_to_cpu(rhdr->residual_count);
376                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
377                 } else
378                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
379         }
380
381         conn->datain_pdus_cnt++;
382         return 0;
383 }
384
385 /**
386  * iscsi_solicit_data_init - initialize first Data-Out
387  * @conn: iscsi connection
388  * @ctask: scsi command task
389  * @r2t: R2T info
390  *
391  * Notes:
392  *      Initialize first Data-Out within this R2T sequence and finds
393  *      proper data_offset within this SCSI command.
394  *
395  *      This function is called with connection lock taken.
396  **/
397 static void
398 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
399                         struct iscsi_r2t_info *r2t)
400 {
401         struct iscsi_data *hdr;
402         struct iscsi_data_task *dtask;
403         struct scsi_cmnd *sc = ctask->sc;
404
405         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
406         BUG_ON(!dtask);
407         hdr = &dtask->hdr;
408         memset(hdr, 0, sizeof(struct iscsi_data));
409         hdr->ttt = r2t->ttt;
410         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
411         r2t->solicit_datasn++;
412         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
413         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
414         hdr->itt = ctask->hdr.itt;
415         hdr->exp_statsn = r2t->exp_statsn;
416         hdr->offset = cpu_to_be32(r2t->data_offset);
417         if (r2t->data_length > conn->max_xmit_dlength) {
418                 hton24(hdr->dlength, conn->max_xmit_dlength);
419                 r2t->data_count = conn->max_xmit_dlength;
420                 hdr->flags = 0;
421         } else {
422                 hton24(hdr->dlength, r2t->data_length);
423                 r2t->data_count = r2t->data_length;
424                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
425         }
426         conn->dataout_pdus_cnt++;
427
428         r2t->sent = 0;
429
430         iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
431                            (u8 *)dtask->hdrext);
432
433         r2t->dtask = dtask;
434
435         if (sc->use_sg) {
436                 int i, sg_count = 0;
437                 struct scatterlist *sg = sc->request_buffer;
438
439                 r2t->sg = NULL;
440                 for (i = 0; i < sc->use_sg; i++, sg += 1) {
441                         /* FIXME: prefetch ? */
442                         if (sg_count + sg->length > r2t->data_offset) {
443                                 int page_offset;
444
445                                 /* sg page found! */
446
447                                 /* offset within this page */
448                                 page_offset = r2t->data_offset - sg_count;
449
450                                 /* fill in this buffer */
451                                 iscsi_buf_init_sg(&r2t->sendbuf, sg);
452                                 r2t->sendbuf.sg.offset += page_offset;
453                                 r2t->sendbuf.sg.length -= page_offset;
454
455                                 /* xmit logic will continue with next one */
456                                 r2t->sg = sg + 1;
457                                 break;
458                         }
459                         sg_count += sg->length;
460                 }
461                 BUG_ON(r2t->sg == NULL);
462         } else
463                 iscsi_buf_init_iov(&ctask->sendbuf,
464                             (char*)sc->request_buffer + r2t->data_offset,
465                             r2t->data_count);
466
467         list_add(&dtask->item, &ctask->dataqueue);
468 }
469
470 /**
471  * iscsi_r2t_rsp - iSCSI R2T Response processing
472  * @conn: iscsi connection
473  * @ctask: scsi command task
474  **/
475 static int
476 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
477 {
478         struct iscsi_r2t_info *r2t;
479         struct iscsi_session *session = conn->session;
480         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
481         int r2tsn = be32_to_cpu(rhdr->r2tsn);
482         int rc;
483
484         if (conn->in.ahslen)
485                 return ISCSI_ERR_AHSLEN;
486
487         if (conn->in.datalen)
488                 return ISCSI_ERR_DATALEN;
489
490         if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
491                 return ISCSI_ERR_R2TSN;
492
493         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
494         if (rc)
495                 return rc;
496
497         /* FIXME: use R2TSN to detect missing R2T */
498
499         /* fill-in new R2T associated with the task */
500         spin_lock(&session->lock);
501         if (!ctask->sc || ctask->mtask ||
502              session->state != ISCSI_STATE_LOGGED_IN) {
503                 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
504                        "recovery...\n", ctask->itt);
505                 spin_unlock(&session->lock);
506                 return 0;
507         }
508         rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
509         BUG_ON(!rc);
510
511         r2t->exp_statsn = rhdr->statsn;
512         r2t->data_length = be32_to_cpu(rhdr->data_length);
513         if (r2t->data_length == 0 ||
514             r2t->data_length > session->max_burst) {
515                 spin_unlock(&session->lock);
516                 return ISCSI_ERR_DATALEN;
517         }
518
519         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
520         if (r2t->data_offset + r2t->data_length > ctask->total_length) {
521                 spin_unlock(&session->lock);
522                 return ISCSI_ERR_DATALEN;
523         }
524
525         r2t->ttt = rhdr->ttt; /* no flip */
526         r2t->solicit_datasn = 0;
527
528         iscsi_solicit_data_init(conn, ctask, r2t);
529
530         ctask->exp_r2tsn = r2tsn + 1;
531         ctask->xmstate |= XMSTATE_SOL_HDR;
532         __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
533         __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
534
535         schedule_work(&conn->xmitwork);
536         conn->r2t_pdus_cnt++;
537         spin_unlock(&session->lock);
538
539         return 0;
540 }
541
542 static int
543 iscsi_hdr_recv(struct iscsi_conn *conn)
544 {
545         int rc = 0;
546         struct iscsi_hdr *hdr;
547         struct iscsi_cmd_task *ctask;
548         struct iscsi_session *session = conn->session;
549         uint32_t cdgst, rdgst = 0;
550
551         hdr = conn->in.hdr;
552
553         /* verify PDU length */
554         conn->in.datalen = ntoh24(hdr->dlength);
555         if (conn->in.datalen > conn->max_recv_dlength) {
556                 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
557                        conn->in.datalen, conn->max_recv_dlength);
558                 return ISCSI_ERR_DATALEN;
559         }
560         conn->data_copied = 0;
561
562         /* read AHS */
563         conn->in.ahslen = hdr->hlength*(4*sizeof(__u16));
564         conn->in.offset += conn->in.ahslen;
565         conn->in.copy -= conn->in.ahslen;
566         if (conn->in.copy < 0) {
567                 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
568                        "%d bytes\n", conn->in.ahslen);
569                 return ISCSI_ERR_AHSLEN;
570         }
571
572         /* calculate read padding */
573         conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
574         if (conn->in.padding) {
575                 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
576                 debug_scsi("read padding %d bytes\n", conn->in.padding);
577         }
578
579         if (conn->hdrdgst_en) {
580                 struct scatterlist sg;
581
582                 sg_init_one(&sg, (u8 *)hdr,
583                             sizeof(struct iscsi_hdr) + conn->in.ahslen);
584                 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
585                 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
586                                      conn->in.ahslen);
587         }
588
589         /* save opcode for later */
590         conn->in.opcode = hdr->opcode;
591
592         /* verify itt (itt encoding: age+cid+itt) */
593         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
594                 if ((hdr->itt & AGE_MASK) !=
595                                 (session->age << AGE_SHIFT)) {
596                         printk(KERN_ERR "iscsi_tcp: received itt %x expected "
597                                 "session age (%x)\n", hdr->itt,
598                                 session->age & AGE_MASK);
599                         return ISCSI_ERR_BAD_ITT;
600                 }
601
602                 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
603                         printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
604                                 "CID (%x)\n", hdr->itt, conn->id);
605                         return ISCSI_ERR_BAD_ITT;
606                 }
607                 conn->in.itt = hdr->itt & ITT_MASK;
608         } else
609                 conn->in.itt = hdr->itt;
610
611         debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
612                   hdr->opcode, conn->in.offset, conn->in.copy,
613                   conn->in.ahslen, conn->in.datalen);
614
615         if (conn->in.itt < session->cmds_max) {
616                 if (conn->hdrdgst_en && cdgst != rdgst) {
617                         printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
618                                "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
619                                cdgst);
620                         return ISCSI_ERR_HDR_DGST;
621                 }
622
623                 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
624
625                 if (!ctask->sc) {
626                         printk(KERN_INFO "iscsi_tcp: dropping ctask with "
627                                "itt 0x%x\n", ctask->itt);
628                         conn->in.datalen = 0; /* force drop */
629                         return 0;
630                 }
631
632                 if (ctask->sc->SCp.phase != session->age) {
633                         printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
634                                 "expected %d\n", ctask->sc->SCp.phase,
635                                 session->age);
636                         return ISCSI_ERR_SESSION_FAILED;
637                 }
638
639                 conn->in.ctask = ctask;
640
641                 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
642                            hdr->opcode, conn->id, (long)ctask->sc,
643                            ctask->itt, conn->in.datalen);
644
645                 switch(conn->in.opcode) {
646                 case ISCSI_OP_SCSI_CMD_RSP:
647                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
648                         if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE)
649                                 rc = iscsi_cmd_rsp(conn, ctask);
650                         else if (!conn->in.datalen)
651                                 rc = iscsi_cmd_rsp(conn, ctask);
652                         else
653                                 /*
654                                  * got sense or response data; copying PDU
655                                  * Header to the connection's header
656                                  * placeholder
657                                  */
658                                 memcpy(&conn->hdr, hdr,
659                                        sizeof(struct iscsi_hdr));
660                         break;
661                 case ISCSI_OP_SCSI_DATA_IN:
662                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
663                         /* save flags for non-exceptional status */
664                         conn->in.flags = hdr->flags;
665                         /* save cmd_status for sense data */
666                         conn->in.cmd_status =
667                                 ((struct iscsi_data_rsp*)hdr)->cmd_status;
668                         rc = iscsi_data_rsp(conn, ctask);
669                         break;
670                 case ISCSI_OP_R2T:
671                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
672                         if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE &&
673                             ctask->sc->sc_data_direction == DMA_TO_DEVICE)
674                                 rc = iscsi_r2t_rsp(conn, ctask);
675                         else
676                                 rc = ISCSI_ERR_PROTO;
677                         break;
678                 case ISCSI_OP_NOOP_IN:
679                 case ISCSI_OP_TEXT_RSP:
680                 case ISCSI_OP_LOGOUT_RSP:
681                 case ISCSI_OP_ASYNC_EVENT:
682                 case ISCSI_OP_REJECT:
683                         rc = iscsi_check_assign_cmdsn(session,
684                                                  (struct iscsi_nopin*)hdr);
685                         if (rc)
686                                 break;
687
688                         /* update ExpStatSN */
689                         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
690                         if (!conn->in.datalen) {
691                                 struct iscsi_mgmt_task *mtask;
692
693                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
694                                                     NULL, 0);
695                                 mtask = (struct iscsi_mgmt_task *)
696                                         session->mgmt_cmds[conn->in.itt -
697                                                 ISCSI_MGMT_ITT_OFFSET];
698                                 if (conn->login_mtask != mtask) {
699                                         spin_lock(&session->lock);
700                                         __kfifo_put(session->mgmtpool.queue,
701                                             (void*)&mtask, sizeof(void*));
702                                         spin_unlock(&session->lock);
703                                 }
704                         }
705                         break;
706                 default:
707                         rc = ISCSI_ERR_BAD_OPCODE;
708                         break;
709                 }
710         } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
711                    conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
712                                         session->mgmtpool_max) {
713                 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
714                                         session->mgmt_cmds[conn->in.itt -
715                                                 ISCSI_MGMT_ITT_OFFSET];
716
717                 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
718                            conn->in.opcode, conn->id, mtask->itt,
719                            conn->in.datalen);
720
721                 switch(conn->in.opcode) {
722                 case ISCSI_OP_LOGIN_RSP:
723                 case ISCSI_OP_TEXT_RSP:
724                         rc = iscsi_check_assign_cmdsn(session,
725                                                  (struct iscsi_nopin*)hdr);
726                         if (rc)
727                                 break;
728
729                         if (!conn->in.datalen) {
730                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
731                                                     NULL, 0);
732                                 if (conn->login_mtask != mtask) {
733                                         spin_lock(&session->lock);
734                                         __kfifo_put(session->mgmtpool.queue,
735                                             (void*)&mtask, sizeof(void*));
736                                         spin_unlock(&session->lock);
737                                 }
738                         }
739                         break;
740                 case ISCSI_OP_SCSI_TMFUNC_RSP:
741                         rc = iscsi_check_assign_cmdsn(session,
742                                                  (struct iscsi_nopin*)hdr);
743                         if (rc)
744                                 break;
745
746                         if (conn->in.datalen || conn->in.ahslen) {
747                                 rc = ISCSI_ERR_PROTO;
748                                 break;
749                         }
750                         conn->tmfrsp_pdus_cnt++;
751                         spin_lock(&session->lock);
752                         if (conn->tmabort_state == TMABORT_INITIAL) {
753                                 __kfifo_put(session->mgmtpool.queue,
754                                                 (void*)&mtask, sizeof(void*));
755                                 conn->tmabort_state =
756                                         ((struct iscsi_tm_rsp *)hdr)->
757                                         response == SCSI_TCP_TM_RESP_COMPLETE ?
758                                                 TMABORT_SUCCESS:TMABORT_FAILED;
759                                 /* unblock eh_abort() */
760                                 wake_up(&conn->ehwait);
761                         }
762                         spin_unlock(&session->lock);
763                         break;
764                 default:
765                         rc = ISCSI_ERR_BAD_OPCODE;
766                         break;
767                 }
768         } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
769                 if (conn->in.opcode == ISCSI_OP_NOOP_IN && !conn->in.datalen) {
770                         rc = iscsi_check_assign_cmdsn(session,
771                                                  (struct iscsi_nopin*)hdr);
772                         if (!rc)
773                                 rc = iscsi_recv_pdu(iscsi_handle(conn),
774                                                     hdr, NULL, 0);
775                 }
776                 else
777                         rc = ISCSI_ERR_BAD_OPCODE;
778         } else
779                 rc = ISCSI_ERR_BAD_ITT;
780
781         return rc;
782 }
783
784 /**
785  * iscsi_ctask_copy - copy skb bits to the destanation cmd task
786  * @conn: iscsi connection
787  * @ctask: scsi command task
788  * @buf: buffer to copy to
789  * @buf_size: size of buffer
790  * @offset: offset within the buffer
791  *
792  * Notes:
793  *      The function calls skb_copy_bits() and updates per-connection and
794  *      per-cmd byte counters.
795  *
796  *      Read counters (in bytes):
797  *
798  *      conn->in.offset         offset within in progress SKB
799  *      conn->in.copy           left to copy from in progress SKB
800  *                              including padding
801  *      conn->in.copied         copied already from in progress SKB
802  *      conn->data_copied       copied already from in progress buffer
803  *      ctask->sent             total bytes sent up to the MidLayer
804  *      ctask->data_count       left to copy from in progress Data-In
805  *      buf_left                left to copy from in progress buffer
806  **/
807 static inline int
808 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
809                 void *buf, int buf_size, int offset)
810 {
811         int buf_left = buf_size - (conn->data_copied + offset);
812         int size = min(conn->in.copy, buf_left);
813         int rc;
814
815         size = min(size, ctask->data_count);
816
817         debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
818                size, conn->in.offset, conn->in.copied);
819
820         BUG_ON(size <= 0);
821         BUG_ON(ctask->sent + size > ctask->total_length);
822
823         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
824                            (char*)buf + (offset + conn->data_copied), size);
825         /* must fit into skb->len */
826         BUG_ON(rc);
827
828         conn->in.offset += size;
829         conn->in.copy -= size;
830         conn->in.copied += size;
831         conn->data_copied += size;
832         ctask->sent += size;
833         ctask->data_count -= size;
834
835         BUG_ON(conn->in.copy < 0);
836         BUG_ON(ctask->data_count < 0);
837
838         if (buf_size != (conn->data_copied + offset)) {
839                 if (!ctask->data_count) {
840                         BUG_ON(buf_size - conn->data_copied < 0);
841                         /* done with this PDU */
842                         return buf_size - conn->data_copied;
843                 }
844                 return -EAGAIN;
845         }
846
847         /* done with this buffer or with both - PDU and buffer */
848         conn->data_copied = 0;
849         return 0;
850 }
851
852 /**
853  * iscsi_tcp_copy - copy skb bits to the destanation buffer
854  * @conn: iscsi connection
855  * @buf: buffer to copy to
856  * @buf_size: number of bytes to copy
857  *
858  * Notes:
859  *      The function calls skb_copy_bits() and updates per-connection
860  *      byte counters.
861  **/
862 static inline int
863 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
864 {
865         int buf_left = buf_size - conn->data_copied;
866         int size = min(conn->in.copy, buf_left);
867         int rc;
868
869         debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
870                size, conn->in.offset, conn->data_copied);
871         BUG_ON(size <= 0);
872
873         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
874                            (char*)buf + conn->data_copied, size);
875         BUG_ON(rc);
876
877         conn->in.offset += size;
878         conn->in.copy -= size;
879         conn->in.copied += size;
880         conn->data_copied += size;
881
882         if (buf_size != conn->data_copied)
883                 return -EAGAIN;
884
885         return 0;
886 }
887
888 static inline void
889 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
890                          int offset, int length)
891 {
892         struct scatterlist temp;
893
894         memcpy(&temp, sg, sizeof(struct scatterlist));
895         temp.offset = offset;
896         temp.length = length;
897         crypto_digest_update(conn->data_rx_tfm, &temp, 1);
898 }
899
900 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
901 {
902         struct iscsi_cmd_task *ctask = conn->in.ctask;
903         struct scsi_cmnd *sc = ctask->sc;
904         struct scatterlist tmp, *sg;
905         int i, offset, rc = 0;
906
907         BUG_ON((void*)ctask != sc->SCp.ptr);
908
909         /*
910          * copying Data-In into the Scsi_Cmnd
911          */
912         if (!sc->use_sg) {
913                 i = ctask->data_count;
914                 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
915                                       sc->request_bufflen, ctask->data_offset);
916                 if (rc == -EAGAIN)
917                         return rc;
918                 if (conn->datadgst_en) {
919                         sg_init_one(&tmp, sc->request_buffer, i);
920                         crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
921                 }
922                 rc = 0;
923                 goto done;
924         }
925
926         offset = ctask->data_offset;
927         sg = sc->request_buffer;
928
929         if (ctask->data_offset)
930                 for (i = 0; i < ctask->sg_count; i++)
931                         offset -= sg[i].length;
932         /* we've passed through partial sg*/
933         if (offset < 0)
934                 offset = 0;
935
936         for (i = ctask->sg_count; i < sc->use_sg; i++) {
937                 char *dest;
938
939                 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
940                 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
941                                       sg[i].length, offset);
942                 kunmap_atomic(dest, KM_SOFTIRQ0);
943                 if (rc == -EAGAIN)
944                         /* continue with the next SKB/PDU */
945                         return rc;
946                 if (!rc) {
947                         if (conn->datadgst_en) {
948                                 if (!offset)
949                                         crypto_digest_update(conn->data_rx_tfm,
950                                                              &sg[i], 1);
951                                 else
952                                         partial_sg_digest_update(conn, &sg[i],
953                                                         sg[i].offset + offset,
954                                                         sg[i].length - offset);
955                         }
956                         offset = 0;
957                         ctask->sg_count++;
958                 }
959
960                 if (!ctask->data_count) {
961                         if (rc && conn->datadgst_en)
962                                 /*
963                                  * data-in is complete, but buffer not...
964                                  */
965                                 partial_sg_digest_update(conn, &sg[i],
966                                                 sg[i].offset, sg[i].length-rc);
967                         rc = 0;
968                         break;
969                 }
970
971                 if (!conn->in.copy)
972                         return -EAGAIN;
973         }
974         BUG_ON(ctask->data_count);
975
976 done:
977         /* check for non-exceptional status */
978         if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
979                 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
980                            (long)sc, sc->result, ctask->itt);
981                 conn->scsirsp_pdus_cnt++;
982                 iscsi_ctask_cleanup(conn, ctask);
983                 sc->scsi_done(sc);
984         }
985
986         return rc;
987 }
988
989 static int
990 iscsi_data_recv(struct iscsi_conn *conn)
991 {
992         struct iscsi_session *session = conn->session;
993         int rc = 0;
994
995         switch(conn->in.opcode) {
996         case ISCSI_OP_SCSI_DATA_IN:
997                 rc = iscsi_scsi_data_in(conn);
998                 break;
999         case ISCSI_OP_SCSI_CMD_RSP: {
1000                 /*
1001                  * SCSI Sense Data:
1002                  * copying the entire Data Segment.
1003                  */
1004                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1005                         rc = -EAGAIN;
1006                         goto exit;
1007                 }
1008
1009                 /*
1010                  * check for sense
1011                  */
1012                 conn->in.hdr = &conn->hdr;
1013                 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1014                 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1015         }
1016         break;
1017         case ISCSI_OP_TEXT_RSP:
1018         case ISCSI_OP_LOGIN_RSP:
1019         case ISCSI_OP_NOOP_IN: {
1020                 struct iscsi_mgmt_task *mtask = NULL;
1021
1022                 if (conn->in.itt != ISCSI_RESERVED_TAG)
1023                         mtask = (struct iscsi_mgmt_task *)
1024                                 session->mgmt_cmds[conn->in.itt -
1025                                         ISCSI_MGMT_ITT_OFFSET];
1026
1027                 /*
1028                  * Collect data segment to the connection's data
1029                  * placeholder
1030                  */
1031                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1032                         rc = -EAGAIN;
1033                         goto exit;
1034                 }
1035
1036                 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1037                                     conn->data, conn->in.datalen);
1038
1039                 if (mtask && conn->login_mtask != mtask) {
1040                         spin_lock(&session->lock);
1041                         __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1042                                     sizeof(void*));
1043                         spin_unlock(&session->lock);
1044                 }
1045         }
1046         break;
1047         default:
1048                 BUG_ON(1);
1049         }
1050 exit:
1051         return rc;
1052 }
1053
1054 /**
1055  * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1056  * @rd_desc: read descriptor
1057  * @skb: socket buffer
1058  * @offset: offset in skb
1059  * @len: skb->len - offset
1060  **/
1061 static int
1062 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1063                 unsigned int offset, size_t len)
1064 {
1065         int rc;
1066         struct iscsi_conn *conn = rd_desc->arg.data;
1067         int start = skb_headlen(skb);
1068         int processed;
1069         char pad[ISCSI_PAD_LEN];
1070         struct scatterlist sg;
1071
1072         /*
1073          * Save current SKB and its offset in the corresponding
1074          * connection context.
1075          */
1076         conn->in.copy = start - offset;
1077         conn->in.offset = offset;
1078         conn->in.skb = skb;
1079         conn->in.len = conn->in.copy;
1080         BUG_ON(conn->in.copy <= 0);
1081         debug_tcp("in %d bytes\n", conn->in.copy);
1082
1083 more:
1084         conn->in.copied = 0;
1085         rc = 0;
1086
1087         if (unlikely(conn->suspend_rx)) {
1088                 debug_tcp("conn %d Rx suspended!\n", conn->id);
1089                 return 0;
1090         }
1091
1092         if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1093             conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1094                 rc = iscsi_hdr_extract(conn);
1095                 if (rc) {
1096                        if (rc == -EAGAIN)
1097                                 goto nomore;
1098                        else {
1099                                 iscsi_conn_failure(conn, rc);
1100                                 return 0;
1101                        }
1102                 }
1103
1104                 /*
1105                  * Verify and process incoming PDU header.
1106                  */
1107                 rc = iscsi_hdr_recv(conn);
1108                 if (!rc && conn->in.datalen) {
1109                         if (conn->datadgst_en &&
1110                             conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1111                                 BUG_ON(!conn->data_rx_tfm);
1112                                 crypto_digest_init(conn->data_rx_tfm);
1113                         }
1114                         conn->in_progress = IN_PROGRESS_DATA_RECV;
1115                 } else if (rc) {
1116                         iscsi_conn_failure(conn, rc);
1117                         return 0;
1118                 }
1119         }
1120
1121         if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1122                 debug_tcp("extra data_recv offset %d copy %d\n",
1123                           conn->in.offset, conn->in.copy);
1124                 if (conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1125                         uint32_t recv_digest;
1126                         skb_copy_bits(conn->in.skb, conn->in.offset,
1127                                       &recv_digest, 4);
1128                         conn->in.offset += 4;
1129                         conn->in.copy -= 4;
1130                         if (recv_digest != conn->in.datadgst) {
1131                                 debug_tcp("iscsi_tcp: data digest error!"
1132                                           "0x%x != 0x%x\n", recv_digest,
1133                                           conn->in.datadgst);
1134                                 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1135                                 return 0;
1136                         } else {
1137                                 debug_tcp("iscsi_tcp: data digest match!"
1138                                           "0x%x == 0x%x\n", recv_digest,
1139                                           conn->in.datadgst);
1140                                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1141                         }
1142                 }
1143         }
1144
1145         if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1146
1147                 debug_tcp("data_recv offset %d copy %d\n",
1148                        conn->in.offset, conn->in.copy);
1149
1150                 rc = iscsi_data_recv(conn);
1151                 if (rc) {
1152                         if (rc == -EAGAIN) {
1153                                 rd_desc->count = conn->in.datalen -
1154                                                 conn->in.ctask->data_count;
1155                                 goto again;
1156                         }
1157                         iscsi_conn_failure(conn, rc);
1158                         return 0;
1159                 }
1160                 conn->in.copy -= conn->in.padding;
1161                 conn->in.offset += conn->in.padding;
1162                 if (conn->datadgst_en &&
1163                     conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1164                         if (conn->in.padding) {
1165                                 debug_tcp("padding -> %d\n", conn->in.padding);
1166                                 memset(pad, 0, conn->in.padding);
1167                                 sg_init_one(&sg, pad, conn->in.padding);
1168                                 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1169                         }
1170                         crypto_digest_final(conn->data_rx_tfm,
1171                                             (u8 *) & conn->in.datadgst);
1172                         debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1173                         conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1174                 } else
1175                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1176         }
1177
1178         debug_tcp("f, processed %d from out of %d padding %d\n",
1179                conn->in.offset - offset, (int)len, conn->in.padding);
1180         BUG_ON(conn->in.offset - offset > len);
1181
1182         if (conn->in.offset - offset != len) {
1183                 debug_tcp("continue to process %d bytes\n",
1184                        (int)len - (conn->in.offset - offset));
1185                 goto more;
1186         }
1187
1188 nomore:
1189         processed = conn->in.offset - offset;
1190         BUG_ON(processed == 0);
1191         return processed;
1192
1193 again:
1194         processed = conn->in.offset - offset;
1195         debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1196                   processed, (int)len, (int)rd_desc->count);
1197         BUG_ON(processed == 0);
1198         BUG_ON(processed > len);
1199
1200         conn->rxdata_octets += processed;
1201         return processed;
1202 }
1203
1204 static void
1205 iscsi_tcp_data_ready(struct sock *sk, int flag)
1206 {
1207         struct iscsi_conn *conn = sk->sk_user_data;
1208         read_descriptor_t rd_desc;
1209
1210         read_lock(&sk->sk_callback_lock);
1211
1212         /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1213         rd_desc.arg.data = conn;
1214         rd_desc.count = 0;
1215         tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1216
1217         read_unlock(&sk->sk_callback_lock);
1218 }
1219
1220 static void
1221 iscsi_tcp_state_change(struct sock *sk)
1222 {
1223         struct iscsi_conn *conn;
1224         struct iscsi_session *session;
1225         void (*old_state_change)(struct sock *);
1226
1227         read_lock(&sk->sk_callback_lock);
1228
1229         conn = (struct iscsi_conn*)sk->sk_user_data;
1230         session = conn->session;
1231
1232         if (sk->sk_state == TCP_CLOSE_WAIT ||
1233             sk->sk_state == TCP_CLOSE) {
1234                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1235                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1236         }
1237
1238         old_state_change = conn->old_state_change;
1239
1240         read_unlock(&sk->sk_callback_lock);
1241
1242         old_state_change(sk);
1243 }
1244
1245 /**
1246  * iscsi_write_space - Called when more output buffer space is available
1247  * @sk: socket space is available for
1248  **/
1249 static void
1250 iscsi_write_space(struct sock *sk)
1251 {
1252         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1253         conn->old_write_space(sk);
1254         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1255         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1256         schedule_work(&conn->xmitwork);
1257 }
1258
1259 static void
1260 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1261 {
1262         struct sock *sk = conn->sock->sk;
1263
1264         /* assign new callbacks */
1265         write_lock_bh(&sk->sk_callback_lock);
1266         sk->sk_user_data = conn;
1267         conn->old_data_ready = sk->sk_data_ready;
1268         conn->old_state_change = sk->sk_state_change;
1269         conn->old_write_space = sk->sk_write_space;
1270         sk->sk_data_ready = iscsi_tcp_data_ready;
1271         sk->sk_state_change = iscsi_tcp_state_change;
1272         sk->sk_write_space = iscsi_write_space;
1273         write_unlock_bh(&sk->sk_callback_lock);
1274 }
1275
1276 static void
1277 iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1278 {
1279         struct sock *sk = conn->sock->sk;
1280
1281         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1282         write_lock_bh(&sk->sk_callback_lock);
1283         sk->sk_user_data    = NULL;
1284         sk->sk_data_ready   = conn->old_data_ready;
1285         sk->sk_state_change = conn->old_state_change;
1286         sk->sk_write_space  = conn->old_write_space;
1287         sk->sk_no_check  = 0;
1288         write_unlock_bh(&sk->sk_callback_lock);
1289 }
1290
1291 /**
1292  * iscsi_send - generic send routine
1293  * @sk: kernel's socket
1294  * @buf: buffer to write from
1295  * @size: actual size to write
1296  * @flags: socket's flags
1297  *
1298  * Notes:
1299  *      depending on buffer will use tcp_sendpage() or tcp_sendmsg().
1300  *      buf->sg.offset == -1 tells us that buffer is non S/G and forces
1301  *      to use tcp_sendmsg().
1302  */
1303 static inline int
1304 iscsi_send(struct socket *sk, struct iscsi_buf *buf, int size, int flags)
1305 {
1306         int res;
1307
1308         if ((int)buf->sg.offset >= 0) {
1309                 int offset = buf->sg.offset + buf->sent;
1310
1311                 /* tcp_sendpage */
1312                 res = sk->ops->sendpage(sk, buf->sg.page, offset, size, flags);
1313         } else {
1314                 struct msghdr msg;
1315
1316                 buf->iov.iov_base = iscsi_buf_iov_base(buf);
1317                 buf->iov.iov_len = size;
1318
1319                 memset(&msg, 0, sizeof(struct msghdr));
1320
1321                 /* tcp_sendmsg */
1322                 res = kernel_sendmsg(sk, &msg, &buf->iov, 1, size);
1323         }
1324
1325         return res;
1326 }
1327
1328 /**
1329  * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1330  * @conn: iscsi connection
1331  * @buf: buffer to write from
1332  * @datalen: lenght of data to be sent after the header
1333  *
1334  * Notes:
1335  *      (Tx, Fast Path)
1336  **/
1337 static inline int
1338 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1339 {
1340         struct socket *sk = conn->sock;
1341         int flags = 0; /* MSG_DONTWAIT; */
1342         int res, size;
1343
1344         size = buf->sg.length - buf->sent;
1345         BUG_ON(buf->sent + size > buf->sg.length);
1346         if (buf->sent + size != buf->sg.length || datalen)
1347                 flags |= MSG_MORE;
1348
1349         res = iscsi_send(sk, buf, size, flags);
1350         debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1351         if (res >= 0) {
1352                 conn->txdata_octets += res;
1353                 buf->sent += res;
1354                 if (size != res)
1355                         return -EAGAIN;
1356                 return 0;
1357         } else if (res == -EAGAIN) {
1358                 conn->sendpage_failures_cnt++;
1359                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1360         } else if (res == -EPIPE)
1361                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1362
1363         return res;
1364 }
1365
1366 /**
1367  * iscsi_sendpage - send one page of iSCSI Data-Out.
1368  * @conn: iscsi connection
1369  * @buf: buffer to write from
1370  * @count: remaining data
1371  * @sent: number of bytes sent
1372  *
1373  * Notes:
1374  *      (Tx, Fast Path)
1375  **/
1376 static inline int
1377 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1378                int *count, int *sent)
1379 {
1380         struct socket *sk = conn->sock;
1381         int flags = 0; /* MSG_DONTWAIT; */
1382         int res, size;
1383
1384         size = buf->sg.length - buf->sent;
1385         BUG_ON(buf->sent + size > buf->sg.length);
1386         if (size > *count)
1387                 size = *count;
1388         if (buf->sent + size != buf->sg.length)
1389                 flags |= MSG_MORE;
1390
1391         res = iscsi_send(sk, buf, size, flags);
1392         debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1393                   size, buf->sent, *count, *sent, res);
1394         if (res >= 0) {
1395                 conn->txdata_octets += res;
1396                 buf->sent += res;
1397                 *count -= res;
1398                 *sent += res;
1399                 if (size != res)
1400                         return -EAGAIN;
1401                 return 0;
1402         } else if (res == -EAGAIN) {
1403                 conn->sendpage_failures_cnt++;
1404                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1405         } else if (res == -EPIPE)
1406                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1407
1408         return res;
1409 }
1410
1411 static inline void
1412 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1413 {
1414         BUG_ON(!conn->data_tx_tfm);
1415         crypto_digest_init(conn->data_tx_tfm);
1416         ctask->digest_count = 4;
1417 }
1418
1419 static inline void
1420 iscsi_buf_data_digest_update(struct iscsi_conn *conn, struct iscsi_buf *buf)
1421 {
1422         struct scatterlist sg;
1423
1424         if (buf->sg.offset != -1)
1425                 crypto_digest_update(conn->data_tx_tfm, &buf->sg, 1);
1426         else {
1427                 sg_init_one(&sg, (char *)buf->sg.page, buf->sg.length);
1428                 crypto_digest_update(conn->data_tx_tfm, &sg, 1);
1429         }
1430 }
1431
1432 static inline int
1433 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1434                         struct iscsi_buf *buf, uint32_t *digest, int final)
1435 {
1436         int rc = 0;
1437         int sent = 0;
1438
1439         if (final)
1440                 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1441
1442         iscsi_buf_init_virt(buf, (char*)digest, 4);
1443         rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1444         if (rc) {
1445                 ctask->datadigest = *digest;
1446                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1447         } else
1448                 ctask->digest_count = 4;
1449         return rc;
1450 }
1451
1452 /**
1453  * iscsi_solicit_data_cont - initialize next Data-Out
1454  * @conn: iscsi connection
1455  * @ctask: scsi command task
1456  * @r2t: R2T info
1457  * @left: bytes left to transfer
1458  *
1459  * Notes:
1460  *      Initialize next Data-Out within this R2T sequence and continue
1461  *      to process next Scatter-Gather element(if any) of this SCSI command.
1462  *
1463  *      Called under connection lock.
1464  **/
1465 static void
1466 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1467                         struct iscsi_r2t_info *r2t, int left)
1468 {
1469         struct iscsi_data *hdr;
1470         struct iscsi_data_task *dtask;
1471         struct scsi_cmnd *sc = ctask->sc;
1472         int new_offset;
1473
1474         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1475         BUG_ON(!dtask);
1476         hdr = &dtask->hdr;
1477         memset(hdr, 0, sizeof(struct iscsi_data));
1478         hdr->ttt = r2t->ttt;
1479         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1480         r2t->solicit_datasn++;
1481         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1482         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1483         hdr->itt = ctask->hdr.itt;
1484         hdr->exp_statsn = r2t->exp_statsn;
1485         new_offset = r2t->data_offset + r2t->sent;
1486         hdr->offset = cpu_to_be32(new_offset);
1487         if (left > conn->max_xmit_dlength) {
1488                 hton24(hdr->dlength, conn->max_xmit_dlength);
1489                 r2t->data_count = conn->max_xmit_dlength;
1490         } else {
1491                 hton24(hdr->dlength, left);
1492                 r2t->data_count = left;
1493                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1494         }
1495         conn->dataout_pdus_cnt++;
1496
1497         iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
1498                            (u8 *)dtask->hdrext);
1499
1500         r2t->dtask = dtask;
1501
1502         if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1503                 BUG_ON(ctask->bad_sg == r2t->sg);
1504                 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1505                 r2t->sg += 1;
1506         } else
1507                 iscsi_buf_init_iov(&ctask->sendbuf,
1508                             (char*)sc->request_buffer + new_offset,
1509                             r2t->data_count);
1510
1511         list_add(&dtask->item, &ctask->dataqueue);
1512 }
1513
1514 static void
1515 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1516 {
1517         struct iscsi_data *hdr;
1518         struct iscsi_data_task *dtask;
1519
1520         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1521         BUG_ON(!dtask);
1522         hdr = &dtask->hdr;
1523         memset(hdr, 0, sizeof(struct iscsi_data));
1524         hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1525         hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1526         ctask->unsol_datasn++;
1527         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1528         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1529         hdr->itt = ctask->hdr.itt;
1530         hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1531         hdr->offset = cpu_to_be32(ctask->total_length -
1532                                   ctask->r2t_data_count -
1533                                   ctask->unsol_count);
1534         if (ctask->unsol_count > conn->max_xmit_dlength) {
1535                 hton24(hdr->dlength, conn->max_xmit_dlength);
1536                 ctask->data_count = conn->max_xmit_dlength;
1537                 hdr->flags = 0;
1538         } else {
1539                 hton24(hdr->dlength, ctask->unsol_count);
1540                 ctask->data_count = ctask->unsol_count;
1541                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1542         }
1543
1544         iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)hdr,
1545                            (u8 *)dtask->hdrext);
1546
1547         list_add(&dtask->item, &ctask->dataqueue);
1548
1549         ctask->dtask = dtask;
1550 }
1551
1552 /**
1553  * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1554  * @conn: iscsi connection
1555  * @ctask: scsi command task
1556  * @sc: scsi command
1557  **/
1558 static void
1559 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1560                 struct scsi_cmnd *sc)
1561 {
1562         struct iscsi_session *session = conn->session;
1563
1564         BUG_ON(__kfifo_len(ctask->r2tqueue));
1565
1566         ctask->sc = sc;
1567         ctask->conn = conn;
1568         ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1569         ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1570         int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1571         ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1572                          (session->age << AGE_SHIFT);
1573         ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1574         ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1575         ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1576         memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1577         memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1578
1579         ctask->mtask = NULL;
1580         ctask->sent = 0;
1581         ctask->sg_count = 0;
1582
1583         ctask->total_length = sc->request_bufflen;
1584
1585         if (sc->sc_data_direction == DMA_TO_DEVICE) {
1586                 ctask->exp_r2tsn = 0;
1587                 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1588                 BUG_ON(ctask->total_length == 0);
1589                 if (sc->use_sg) {
1590                         struct scatterlist *sg = sc->request_buffer;
1591
1592                         iscsi_buf_init_sg(&ctask->sendbuf,
1593                                           &sg[ctask->sg_count++]);
1594                         ctask->sg = sg;
1595                         ctask->bad_sg = sg + sc->use_sg;
1596                 } else {
1597                         iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1598                                         sc->request_bufflen);
1599                 }
1600
1601                 /*
1602                  * Write counters:
1603                  *
1604                  *      imm_count       bytes to be sent right after
1605                  *                      SCSI PDU Header
1606                  *
1607                  *      unsol_count     bytes(as Data-Out) to be sent
1608                  *                      without R2T ack right after
1609                  *                      immediate data
1610                  *
1611                  *      r2t_data_count  bytes to be sent via R2T ack's
1612                  *
1613                  *      pad_count       bytes to be sent as zero-padding
1614                  */
1615                 ctask->imm_count = 0;
1616                 ctask->unsol_count = 0;
1617                 ctask->unsol_datasn = 0;
1618                 ctask->xmstate = XMSTATE_W_HDR;
1619                 /* calculate write padding */
1620                 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1621                 if (ctask->pad_count) {
1622                         ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1623                         debug_scsi("write padding %d bytes\n",
1624                                 ctask->pad_count);
1625                         ctask->xmstate |= XMSTATE_W_PAD;
1626                 }
1627                 if (session->imm_data_en) {
1628                         if (ctask->total_length >= session->first_burst)
1629                                 ctask->imm_count = min(session->first_burst,
1630                                                         conn->max_xmit_dlength);
1631                         else
1632                                 ctask->imm_count = min(ctask->total_length,
1633                                                         conn->max_xmit_dlength);
1634                         hton24(ctask->hdr.dlength, ctask->imm_count);
1635                         ctask->xmstate |= XMSTATE_IMM_DATA;
1636                 } else
1637                         zero_data(ctask->hdr.dlength);
1638
1639                 if (!session->initial_r2t_en)
1640                         ctask->unsol_count = min(session->first_burst,
1641                                 ctask->total_length) - ctask->imm_count;
1642                 if (!ctask->unsol_count)
1643                         /* No unsolicit Data-Out's */
1644                         ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1645                 else
1646                         ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1647
1648                 ctask->r2t_data_count = ctask->total_length -
1649                                     ctask->imm_count -
1650                                     ctask->unsol_count;
1651
1652                 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1653                            "r2t_data %d]\n",
1654                            ctask->itt, ctask->total_length, ctask->imm_count,
1655                            ctask->unsol_count, ctask->r2t_data_count);
1656         } else {
1657                 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1658                 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1659                         ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1660                 ctask->datasn = 0;
1661                 ctask->xmstate = XMSTATE_R_HDR;
1662                 zero_data(ctask->hdr.dlength);
1663         }
1664
1665         iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)&ctask->hdr,
1666                             (u8 *)ctask->hdrext);
1667         conn->scsicmd_pdus_cnt++;
1668 }
1669
1670 /**
1671  * iscsi_mtask_xmit - xmit management(immediate) task
1672  * @conn: iscsi connection
1673  * @mtask: task management task
1674  *
1675  * Notes:
1676  *      The function can return -EAGAIN in which case caller must
1677  *      call it again later, or recover. '0' return code means successful
1678  *      xmit.
1679  *
1680  *      Management xmit state machine consists of two states:
1681  *              IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1682  *              IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1683  **/
1684 static int
1685 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1686 {
1687
1688         debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1689                 conn->id, mtask->xmstate, mtask->itt);
1690
1691         if (mtask->xmstate & XMSTATE_IMM_HDR) {
1692                 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1693                 if (mtask->data_count)
1694                         mtask->xmstate |= XMSTATE_IMM_DATA;
1695                 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1696                         mtask->xmstate |= XMSTATE_IMM_HDR;
1697                         if (mtask->data_count)
1698                                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1699                         return -EAGAIN;
1700                 }
1701         }
1702
1703         if (mtask->xmstate & XMSTATE_IMM_DATA) {
1704                 BUG_ON(!mtask->data_count);
1705                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1706                 /* FIXME: implement.
1707                  * Virtual buffer could be spreaded across multiple pages...
1708                  */
1709                 do {
1710                         if (iscsi_sendpage(conn, &mtask->sendbuf,
1711                                    &mtask->data_count, &mtask->sent)) {
1712                                 mtask->xmstate |= XMSTATE_IMM_DATA;
1713                                 return -EAGAIN;
1714                         }
1715                 } while (mtask->data_count);
1716         }
1717
1718         BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1719         return 0;
1720 }
1721
1722 static inline int
1723 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1724 {
1725         ctask->xmstate &= ~XMSTATE_R_HDR;
1726         if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1727                 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1728                 return 0; /* wait for Data-In */
1729         }
1730         ctask->xmstate |= XMSTATE_R_HDR;
1731         return -EAGAIN;
1732 }
1733
1734 static inline int
1735 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1736 {
1737         ctask->xmstate &= ~XMSTATE_W_HDR;
1738         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1739                 ctask->xmstate |= XMSTATE_W_HDR;
1740                 return -EAGAIN;
1741         }
1742         return 0;
1743 }
1744
1745 static inline int
1746 handle_xmstate_data_digest(struct iscsi_conn *conn,
1747                            struct iscsi_cmd_task *ctask)
1748 {
1749         ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1750         debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1751         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1752                                     &ctask->datadigest, 0)) {
1753                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1754                 debug_tcp("resent data digest 0x%x fail!\n",
1755                           ctask->datadigest);
1756                 return -EAGAIN;
1757         }
1758         return 0;
1759 }
1760
1761 static inline int
1762 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1763 {
1764         BUG_ON(!ctask->imm_count);
1765         ctask->xmstate &= ~XMSTATE_IMM_DATA;
1766
1767         if (conn->datadgst_en) {
1768                 iscsi_data_digest_init(conn, ctask);
1769                 ctask->immdigest = 0;
1770         }
1771
1772         for (;;) {
1773                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1774                                    &ctask->sent)) {
1775                         ctask->xmstate |= XMSTATE_IMM_DATA;
1776                         if (conn->datadgst_en) {
1777                                 crypto_digest_final(conn->data_tx_tfm,
1778                                                 (u8*)&ctask->immdigest);
1779                                 debug_tcp("tx imm sendpage fail 0x%x\n",
1780                                           ctask->datadigest);
1781                         }
1782                         return -EAGAIN;
1783                 }
1784                 if (conn->datadgst_en)
1785                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1786
1787                 if (!ctask->imm_count)
1788                         break;
1789                 iscsi_buf_init_sg(&ctask->sendbuf,
1790                                   &ctask->sg[ctask->sg_count++]);
1791         }
1792
1793         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1794                 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1795                                             &ctask->immdigest, 1)) {
1796                         debug_tcp("sending imm digest 0x%x fail!\n",
1797                                   ctask->immdigest);
1798                         return -EAGAIN;
1799                 }
1800                 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1801         }
1802
1803         return 0;
1804 }
1805
1806 static inline int
1807 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1808 {
1809         struct iscsi_data_task *dtask;
1810
1811         ctask->xmstate |= XMSTATE_UNS_DATA;
1812         if (ctask->xmstate & XMSTATE_UNS_INIT) {
1813                 iscsi_unsolicit_data_init(conn, ctask);
1814                 BUG_ON(!ctask->dtask);
1815                 dtask = ctask->dtask;
1816
1817                 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1818         }
1819         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1820                 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1821                 ctask->xmstate |= XMSTATE_UNS_HDR;
1822                 return -EAGAIN;
1823         }
1824
1825         debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1826                    ctask->itt, ctask->unsol_count, ctask->sent);
1827         return 0;
1828 }
1829
1830 static inline int
1831 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1832 {
1833         struct iscsi_data_task *dtask = ctask->dtask;
1834
1835         BUG_ON(!ctask->data_count);
1836         ctask->xmstate &= ~XMSTATE_UNS_DATA;
1837
1838         if (conn->datadgst_en) {
1839                 iscsi_data_digest_init(conn, ctask);
1840                 dtask->digest = 0;
1841         }
1842
1843         for (;;) {
1844                 int start = ctask->sent;
1845
1846                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1847                                    &ctask->sent)) {
1848                         ctask->unsol_count -= ctask->sent - start;
1849                         ctask->xmstate |= XMSTATE_UNS_DATA;
1850                         /* will continue with this ctask later.. */
1851                         if (conn->datadgst_en) {
1852                                 crypto_digest_final(conn->data_tx_tfm,
1853                                                 (u8 *)&dtask->digest);
1854                                 debug_tcp("tx uns data fail 0x%x\n",
1855                                           dtask->digest);
1856                         }
1857                         return -EAGAIN;
1858                 }
1859
1860                 BUG_ON(ctask->sent > ctask->total_length);
1861                 ctask->unsol_count -= ctask->sent - start;
1862
1863                 /*
1864                  * XXX:we may run here with un-initial sendbuf.
1865                  * so pass it
1866                  */
1867                 if (conn->datadgst_en && ctask->sent - start > 0)
1868                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1869
1870                 if (!ctask->data_count)
1871                         break;
1872                 iscsi_buf_init_sg(&ctask->sendbuf,
1873                                   &ctask->sg[ctask->sg_count++]);
1874         }
1875         BUG_ON(ctask->unsol_count < 0);
1876
1877         /*
1878          * Done with the Data-Out. Next, check if we need
1879          * to send another unsolicited Data-Out.
1880          */
1881         if (ctask->unsol_count) {
1882                 if (conn->datadgst_en) {
1883                         if (iscsi_digest_final_send(conn, ctask,
1884                                                     &dtask->digestbuf,
1885                                                     &dtask->digest, 1)) {
1886                                 debug_tcp("send uns digest 0x%x fail\n",
1887                                           dtask->digest);
1888                                 return -EAGAIN;
1889                         }
1890                         debug_tcp("sending uns digest 0x%x, more uns\n",
1891                                   dtask->digest);
1892                 }
1893                 ctask->xmstate |= XMSTATE_UNS_INIT;
1894                 return 1;
1895         }
1896
1897         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1898                 if (iscsi_digest_final_send(conn, ctask,
1899                                             &dtask->digestbuf,
1900                                             &dtask->digest, 1)) {
1901                         debug_tcp("send last uns digest 0x%x fail\n",
1902                                    dtask->digest);
1903                         return -EAGAIN;
1904                 }
1905                 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1906         }
1907
1908         return 0;
1909 }
1910
1911 static inline int
1912 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1913 {
1914         struct iscsi_session *session = conn->session;
1915         struct iscsi_r2t_info *r2t = ctask->r2t;
1916         struct iscsi_data_task *dtask = r2t->dtask;
1917         int left;
1918
1919         ctask->xmstate &= ~XMSTATE_SOL_DATA;
1920         ctask->dtask = dtask;
1921
1922         if (conn->datadgst_en) {
1923                 iscsi_data_digest_init(conn, ctask);
1924                 dtask->digest = 0;
1925         }
1926 solicit_again:
1927         /*
1928          * send Data-Out whitnin this R2T sequence.
1929          */
1930         if (!r2t->data_count)
1931                 goto data_out_done;
1932
1933         if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1934                 ctask->xmstate |= XMSTATE_SOL_DATA;
1935                 /* will continue with this ctask later.. */
1936                 if (conn->datadgst_en) {
1937                         crypto_digest_final(conn->data_tx_tfm,
1938                                           (u8 *)&dtask->digest);
1939                         debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1940                 }
1941                 return -EAGAIN;
1942         }
1943
1944         BUG_ON(r2t->data_count < 0);
1945         if (conn->datadgst_en)
1946                 iscsi_buf_data_digest_update(conn, &r2t->sendbuf);
1947
1948         if (r2t->data_count) {
1949                 BUG_ON(ctask->sc->use_sg == 0);
1950                 if (!iscsi_buf_left(&r2t->sendbuf)) {
1951                         BUG_ON(ctask->bad_sg == r2t->sg);
1952                         iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1953                         r2t->sg += 1;
1954                 }
1955                 goto solicit_again;
1956         }
1957
1958 data_out_done:
1959         /*
1960          * Done with this Data-Out. Next, check if we have
1961          * to send another Data-Out for this R2T.
1962          */
1963         BUG_ON(r2t->data_length - r2t->sent < 0);
1964         left = r2t->data_length - r2t->sent;
1965         if (left) {
1966                 if (conn->datadgst_en) {
1967                         if (iscsi_digest_final_send(conn, ctask,
1968                                                     &dtask->digestbuf,
1969                                                     &dtask->digest, 1)) {
1970                                 debug_tcp("send r2t data digest 0x%x"
1971                                           "fail\n", dtask->digest);
1972                                 return -EAGAIN;
1973                         }
1974                         debug_tcp("r2t data send digest 0x%x\n",
1975                                   dtask->digest);
1976                 }
1977                 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1978                 ctask->xmstate |= XMSTATE_SOL_DATA;
1979                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
1980                 return 1;
1981         }
1982
1983         /*
1984          * Done with this R2T. Check if there are more
1985          * outstanding R2Ts ready to be processed.
1986          */
1987         BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
1988         if (conn->datadgst_en) {
1989                 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1990                                             &dtask->digest, 1)) {
1991                         debug_tcp("send last r2t data digest 0x%x"
1992                                   "fail\n", dtask->digest);
1993                         return -EAGAIN;
1994                 }
1995                 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1996         }
1997
1998         ctask->r2t_data_count -= r2t->data_length;
1999         ctask->r2t = NULL;
2000         spin_lock_bh(&session->lock);
2001         __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2002         spin_unlock_bh(&session->lock);
2003         if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2004                 ctask->r2t = r2t;
2005                 ctask->xmstate |= XMSTATE_SOL_DATA;
2006                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2007                 return 1;
2008         }
2009
2010         return 0;
2011 }
2012
2013 static inline int
2014 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2015 {
2016         struct iscsi_data_task *dtask = ctask->dtask;
2017         int sent;
2018
2019         ctask->xmstate &= ~XMSTATE_W_PAD;
2020         iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2021                             ctask->pad_count);
2022         if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2023                 ctask->xmstate |= XMSTATE_W_PAD;
2024                 return -EAGAIN;
2025         }
2026
2027         if (conn->datadgst_en) {
2028                 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
2029                 /* imm data? */
2030                 if (!dtask) {
2031                         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2032                                                     &ctask->immdigest, 1)) {
2033                                 debug_tcp("send padding digest 0x%x"
2034                                           "fail!\n", ctask->immdigest);
2035                                 return -EAGAIN;
2036                         }
2037                         debug_tcp("done with padding, digest 0x%x\n",
2038                                   ctask->datadigest);
2039                 } else {
2040                         if (iscsi_digest_final_send(conn, ctask,
2041                                                     &dtask->digestbuf,
2042                                                     &dtask->digest, 1)) {
2043                                 debug_tcp("send padding digest 0x%x"
2044                                           "fail\n", dtask->digest);
2045                                 return -EAGAIN;
2046                         }
2047                         debug_tcp("done with padding, digest 0x%x\n",
2048                                   dtask->digest);
2049                 }
2050         }
2051
2052         return 0;
2053 }
2054
2055 static int
2056 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2057 {
2058         int rc = 0;
2059
2060         debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2061                 conn->id, ctask->xmstate, ctask->itt);
2062
2063         /*
2064          * serialize with TMF AbortTask
2065          */
2066         if (ctask->mtask)
2067                 return rc;
2068
2069         if (ctask->xmstate & XMSTATE_R_HDR) {
2070                 rc = handle_xmstate_r_hdr(conn, ctask);
2071                 return rc;
2072         }
2073
2074         if (ctask->xmstate & XMSTATE_W_HDR) {
2075                 rc = handle_xmstate_w_hdr(conn, ctask);
2076                 if (rc)
2077                         return rc;
2078         }
2079
2080         /* XXX: for data digest xmit recover */
2081         if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2082                 rc = handle_xmstate_data_digest(conn, ctask);
2083                 if (rc)
2084                         return rc;
2085         }
2086
2087         if (ctask->xmstate & XMSTATE_IMM_DATA) {
2088                 rc = handle_xmstate_imm_data(conn, ctask);
2089                 if (rc)
2090                         return rc;
2091         }
2092
2093         if (ctask->xmstate & XMSTATE_UNS_HDR) {
2094                 BUG_ON(!ctask->unsol_count);
2095                 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2096 unsolicit_head_again:
2097                 rc = handle_xmstate_uns_hdr(conn, ctask);
2098                 if (rc)
2099                         return rc;
2100         }
2101
2102         if (ctask->xmstate & XMSTATE_UNS_DATA) {
2103                 rc = handle_xmstate_uns_data(conn, ctask);
2104                 if (rc == 1)
2105                         goto unsolicit_head_again;
2106                 else if (rc)
2107                         return rc;
2108                 goto done;
2109         }
2110
2111         if (ctask->xmstate & XMSTATE_SOL_HDR) {
2112                 struct iscsi_r2t_info *r2t;
2113
2114                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2115                 ctask->xmstate |= XMSTATE_SOL_DATA;
2116                 if (!ctask->r2t)
2117                         __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2118                                     sizeof(void*));
2119 solicit_head_again:
2120                 r2t = ctask->r2t;
2121
2122                 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2123                         ctask->xmstate &= ~XMSTATE_SOL_DATA;
2124                         ctask->xmstate |= XMSTATE_SOL_HDR;
2125                         return -EAGAIN;
2126                 }
2127
2128                 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2129                         r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2130                         r2t->sent);
2131         }
2132
2133         if (ctask->xmstate & XMSTATE_SOL_DATA) {
2134                 rc = handle_xmstate_sol_data(conn, ctask);
2135                 if (rc == 1)
2136                         goto solicit_head_again;
2137                 if (rc)
2138                         return rc;
2139         }
2140
2141 done:
2142         /*
2143          * Last thing to check is whether we need to send write
2144          * padding. Note that we check for xmstate equality, not just the bit.
2145          */
2146         if (ctask->xmstate == XMSTATE_W_PAD)
2147                 rc = handle_xmstate_w_pad(conn, ctask);
2148
2149         return rc;
2150 }
2151
2152 /**
2153  * iscsi_data_xmit - xmit any command into the scheduled connection
2154  * @conn: iscsi connection
2155  *
2156  * Notes:
2157  *      The function can return -EAGAIN in which case the caller must
2158  *      re-schedule it again later or recover. '0' return code means
2159  *      successful xmit.
2160  **/
2161 static int
2162 iscsi_data_xmit(struct iscsi_conn *conn)
2163 {
2164         if (unlikely(conn->suspend_tx)) {
2165                 debug_tcp("conn %d Tx suspended!\n", conn->id);
2166                 return 0;
2167         }
2168
2169         /*
2170          * Transmit in the following order:
2171          *
2172          * 1) un-finished xmit (ctask or mtask)
2173          * 2) immediate control PDUs
2174          * 3) write data
2175          * 4) SCSI commands
2176          * 5) non-immediate control PDUs
2177          *
2178          * No need to lock around __kfifo_get as long as
2179          * there's one producer and one consumer.
2180          */
2181
2182         BUG_ON(conn->ctask && conn->mtask);
2183
2184         if (conn->ctask) {
2185                 if (iscsi_ctask_xmit(conn, conn->ctask))
2186                         goto again;
2187                 /* done with this in-progress ctask */
2188                 conn->ctask = NULL;
2189         }
2190         if (conn->mtask) {
2191                 if (iscsi_mtask_xmit(conn, conn->mtask))
2192                         goto again;
2193                 /* done with this in-progress mtask */
2194                 conn->mtask = NULL;
2195         }
2196
2197         /* process immediate first */
2198         if (unlikely(__kfifo_len(conn->immqueue))) {
2199                 struct iscsi_session *session = conn->session;
2200                 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2201                                    sizeof(void*))) {
2202                         if (iscsi_mtask_xmit(conn, conn->mtask))
2203                                 goto again;
2204
2205                         if (conn->mtask->hdr.itt ==
2206                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2207                                 spin_lock_bh(&session->lock);
2208                                 __kfifo_put(session->mgmtpool.queue,
2209                                             (void*)&conn->mtask, sizeof(void*));
2210                                 spin_unlock_bh(&session->lock);
2211                         }
2212                 }
2213                 /* done with this mtask */
2214                 conn->mtask = NULL;
2215         }
2216
2217         /* process write queue */
2218         while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2219                            sizeof(void*))) {
2220                 if (iscsi_ctask_xmit(conn, conn->ctask))
2221                         goto again;
2222         }
2223
2224         /* process command queue */
2225         while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2226                            sizeof(void*))) {
2227                 if (iscsi_ctask_xmit(conn, conn->ctask))
2228                         goto again;
2229         }
2230         /* done with this ctask */
2231         conn->ctask = NULL;
2232
2233         /* process the rest control plane PDUs, if any */
2234         if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2235                 struct iscsi_session *session = conn->session;
2236
2237                 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2238                                    sizeof(void*))) {
2239                         if (iscsi_mtask_xmit(conn, conn->mtask))
2240                                 goto again;
2241
2242                         if (conn->mtask->hdr.itt ==
2243                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2244                                 spin_lock_bh(&session->lock);
2245                                 __kfifo_put(session->mgmtpool.queue,
2246                                             (void*)&conn->mtask,
2247                                             sizeof(void*));
2248                                 spin_unlock_bh(&session->lock);
2249                         }
2250                 }
2251                 /* done with this mtask */
2252                 conn->mtask = NULL;
2253         }
2254
2255         return 0;
2256
2257 again:
2258         if (unlikely(conn->suspend_tx))
2259                 return 0;
2260
2261         return -EAGAIN;
2262 }
2263
2264 static void
2265 iscsi_xmitworker(void *data)
2266 {
2267         struct iscsi_conn *conn = data;
2268
2269         /*
2270          * serialize Xmit worker on a per-connection basis.
2271          */
2272         down(&conn->xmitsema);
2273         if (iscsi_data_xmit(conn))
2274                 schedule_work(&conn->xmitwork);
2275         up(&conn->xmitsema);
2276 }
2277
2278 #define FAILURE_BAD_HOST                1
2279 #define FAILURE_SESSION_FAILED          2
2280 #define FAILURE_SESSION_FREED           3
2281 #define FAILURE_WINDOW_CLOSED           4
2282 #define FAILURE_SESSION_TERMINATE       5
2283
2284 static int
2285 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2286 {
2287         struct Scsi_Host *host;
2288         int reason = 0;
2289         struct iscsi_session *session;
2290         struct iscsi_conn *conn = NULL;
2291         struct iscsi_cmd_task *ctask = NULL;
2292
2293         sc->scsi_done = done;
2294         sc->result = 0;
2295
2296         host = sc->device->host;
2297         session = iscsi_hostdata(host->hostdata);
2298         BUG_ON(host != session->host);
2299
2300         spin_lock(&session->lock);
2301
2302         if (session->state != ISCSI_STATE_LOGGED_IN) {
2303                 if (session->state == ISCSI_STATE_FAILED) {
2304                         reason = FAILURE_SESSION_FAILED;
2305                         goto reject;
2306                 } else if (session->state == ISCSI_STATE_TERMINATE) {
2307                         reason = FAILURE_SESSION_TERMINATE;
2308                         goto fault;
2309                 }
2310                 reason = FAILURE_SESSION_FREED;
2311                 goto fault;
2312         }
2313
2314         /*
2315          * Check for iSCSI window and take care of CmdSN wrap-around
2316          */
2317         if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2318                 reason = FAILURE_WINDOW_CLOSED;
2319                 goto reject;
2320         }
2321
2322         conn = session->leadconn;
2323
2324         __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2325         BUG_ON(ctask->sc);
2326
2327         sc->SCp.phase = session->age;
2328         sc->SCp.ptr = (char*)ctask;
2329         iscsi_cmd_init(conn, ctask, sc);
2330
2331         __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2332         debug_scsi(
2333                "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2334                 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2335                 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2336                 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2337         spin_unlock(&session->lock);
2338
2339         if (!in_interrupt() && !down_trylock(&conn->xmitsema)) {
2340                 spin_unlock_irq(host->host_lock);
2341                 if (iscsi_data_xmit(conn))
2342                         schedule_work(&conn->xmitwork);
2343                 up(&conn->xmitsema);
2344                 spin_lock_irq(host->host_lock);
2345         } else
2346                 schedule_work(&conn->xmitwork);
2347
2348         return 0;
2349
2350 reject:
2351         spin_unlock(&session->lock);
2352         debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2353         return SCSI_MLQUEUE_HOST_BUSY;
2354
2355 fault:
2356         spin_unlock(&session->lock);
2357         printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2358                sc->cmnd[0], reason);
2359         sc->sense_buffer[0] = 0x70;
2360         sc->sense_buffer[2] = NOT_READY;
2361         sc->sense_buffer[7] = 0x6;
2362         sc->sense_buffer[12] = 0x08;
2363         sc->sense_buffer[13] = 0x00;
2364         sc->result = (DID_NO_CONNECT << 16);
2365         sc->resid = sc->request_bufflen;
2366         sc->scsi_done(sc);
2367         return 0;
2368 }
2369
2370 static int
2371 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2372 {
2373         int i;
2374
2375         *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2376         if (*items == NULL)
2377                 return -ENOMEM;
2378
2379         q->max = max;
2380         q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2381         if (q->pool == NULL) {
2382                 kfree(*items);
2383                 return -ENOMEM;
2384         }
2385
2386         q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2387                               GFP_KERNEL, NULL);
2388         if (q->queue == ERR_PTR(-ENOMEM)) {
2389                 kfree(q->pool);
2390                 kfree(*items);
2391                 return -ENOMEM;
2392         }
2393
2394         for (i = 0; i < max; i++) {
2395                 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2396                 if (q->pool[i] == NULL) {
2397                         int j;
2398
2399                         for (j = 0; j < i; j++)
2400                                 kfree(q->pool[j]);
2401
2402                         kfifo_free(q->queue);
2403                         kfree(q->pool);
2404                         kfree(*items);
2405                         return -ENOMEM;
2406                 }
2407                 memset(q->pool[i], 0, item_size);
2408                 (*items)[i] = q->pool[i];
2409                 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2410         }
2411         return 0;
2412 }
2413
2414 static void
2415 iscsi_pool_free(struct iscsi_queue *q, void **items)
2416 {
2417         int i;
2418
2419         for (i = 0; i < q->max; i++)
2420                 kfree(items[i]);
2421         kfree(q->pool);
2422         kfree(items);
2423 }
2424
2425 static iscsi_connh_t
2426 iscsi_conn_create(iscsi_sessionh_t sessionh, uint32_t conn_idx)
2427 {
2428         struct iscsi_session *session = iscsi_ptr(sessionh);
2429         struct iscsi_conn *conn = NULL;
2430
2431         conn = kmalloc(sizeof(struct iscsi_conn), GFP_KERNEL);
2432         if (conn == NULL)
2433                 goto conn_alloc_fail;
2434         memset(conn, 0, sizeof(struct iscsi_conn));
2435
2436         conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2437         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2438         conn->id = conn_idx;
2439         conn->exp_statsn = 0;
2440         conn->tmabort_state = TMABORT_INITIAL;
2441
2442         /* initial operational parameters */
2443         conn->hdr_size = sizeof(struct iscsi_hdr);
2444         conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2445         conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2446
2447         spin_lock_init(&conn->lock);
2448
2449         /* initialize general xmit PDU commands queue */
2450         conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2451                                         GFP_KERNEL, NULL);
2452         if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2453                 goto xmitqueue_alloc_fail;
2454
2455         /* initialize write response PDU commands queue */
2456         conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2457                                         GFP_KERNEL, NULL);
2458         if (conn->writequeue == ERR_PTR(-ENOMEM))
2459                 goto writequeue_alloc_fail;
2460
2461         /* initialize general immediate & non-immediate PDU commands queue */
2462         conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2463                                         GFP_KERNEL, NULL);
2464         if (conn->immqueue == ERR_PTR(-ENOMEM))
2465                 goto immqueue_alloc_fail;
2466
2467         conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2468                                         GFP_KERNEL, NULL);
2469         if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2470                 goto mgmtqueue_alloc_fail;
2471
2472         INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2473
2474         /* allocate login_mtask used for the login/text sequences */
2475         spin_lock_bh(&session->lock);
2476         if (!__kfifo_get(session->mgmtpool.queue,
2477                          (void*)&conn->login_mtask,
2478                          sizeof(void*))) {
2479                 spin_unlock_bh(&session->lock);
2480                 goto login_mtask_alloc_fail;
2481         }
2482         spin_unlock_bh(&session->lock);
2483
2484         /* allocate initial PDU receive place holder */
2485         if (conn->data_size <= PAGE_SIZE)
2486                 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2487         else
2488                 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2489                                         get_order(conn->data_size));
2490         if (!conn->data)
2491                 goto max_recv_dlenght_alloc_fail;
2492
2493         init_timer(&conn->tmabort_timer);
2494         init_MUTEX(&conn->xmitsema);
2495         init_waitqueue_head(&conn->ehwait);
2496
2497         return iscsi_handle(conn);
2498
2499 max_recv_dlenght_alloc_fail:
2500         spin_lock_bh(&session->lock);
2501         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2502                     sizeof(void*));
2503         spin_unlock_bh(&session->lock);
2504 login_mtask_alloc_fail:
2505         kfifo_free(conn->mgmtqueue);
2506 mgmtqueue_alloc_fail:
2507         kfifo_free(conn->immqueue);
2508 immqueue_alloc_fail:
2509         kfifo_free(conn->writequeue);
2510 writequeue_alloc_fail:
2511         kfifo_free(conn->xmitqueue);
2512 xmitqueue_alloc_fail:
2513         kfree(conn);
2514 conn_alloc_fail:
2515         return iscsi_handle(NULL);
2516 }
2517
2518 static void
2519 iscsi_conn_destroy(iscsi_connh_t connh)
2520 {
2521         struct iscsi_conn *conn = iscsi_ptr(connh);
2522         struct iscsi_session *session = conn->session;
2523
2524         down(&conn->xmitsema);
2525         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2526         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2527                 struct sock *sk = conn->sock->sk;
2528
2529                 /*
2530                  * conn_start() has never been called!
2531                  * need to cleanup the socket.
2532                  */
2533                 write_lock_bh(&sk->sk_callback_lock);
2534                 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2535                 write_unlock_bh(&sk->sk_callback_lock);
2536
2537                 sock_hold(conn->sock->sk);
2538                 iscsi_conn_restore_callbacks(conn);
2539                 sock_put(conn->sock->sk);
2540                 sock_release(conn->sock);
2541                 conn->sock = NULL;
2542         }
2543
2544         spin_lock_bh(&session->lock);
2545         conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2546         if (session->leadconn == conn) {
2547                 /*
2548                  * leading connection? then give up on recovery.
2549                  */
2550                 session->state = ISCSI_STATE_TERMINATE;
2551                 wake_up(&conn->ehwait);
2552         }
2553         spin_unlock_bh(&session->lock);
2554
2555         up(&conn->xmitsema);
2556
2557         /*
2558          * Block until all in-progress commands for this connection
2559          * time out or fail.
2560          */
2561         for (;;) {
2562                 spin_lock_bh(&conn->lock);
2563                 if (!session->host->host_busy) { /* OK for ERL == 0 */
2564                         spin_unlock_bh(&conn->lock);
2565                         break;
2566                 }
2567                 spin_unlock_bh(&conn->lock);
2568                 msleep_interruptible(500);
2569                 printk("conn_destroy(): host_busy %d host_failed %d\n",
2570                         session->host->host_busy, session->host->host_failed);
2571                 /*
2572                  * force eh_abort() to unblock
2573                  */
2574                 wake_up(&conn->ehwait);
2575         }
2576
2577         /* now free crypto */
2578         if (conn->hdrdgst_en || conn->datadgst_en) {
2579                 if (conn->tx_tfm)
2580                         crypto_free_tfm(conn->tx_tfm);
2581                 if (conn->rx_tfm)
2582                         crypto_free_tfm(conn->rx_tfm);
2583                 if (conn->data_tx_tfm)
2584                         crypto_free_tfm(conn->data_tx_tfm);
2585                 if (conn->data_rx_tfm)
2586                         crypto_free_tfm(conn->data_rx_tfm);
2587         }
2588
2589         /* free conn->data, size = MaxRecvDataSegmentLength */
2590         if (conn->data_size <= PAGE_SIZE)
2591                 kfree(conn->data);
2592         else
2593                 free_pages((unsigned long)conn->data,
2594                                         get_order(conn->data_size));
2595
2596         spin_lock_bh(&session->lock);
2597         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2598                     sizeof(void*));
2599         list_del(&conn->item);
2600         if (list_empty(&session->connections))
2601                 session->leadconn = NULL;
2602         if (session->leadconn && session->leadconn == conn)
2603                 session->leadconn = container_of(session->connections.next,
2604                         struct iscsi_conn, item);
2605
2606         if (session->leadconn == NULL)
2607                 /* none connections exits.. reset sequencing */
2608                 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2609         spin_unlock_bh(&session->lock);
2610
2611         kfifo_free(conn->xmitqueue);
2612         kfifo_free(conn->writequeue);
2613         kfifo_free(conn->immqueue);
2614         kfifo_free(conn->mgmtqueue);
2615         kfree(conn);
2616 }
2617
2618 static int
2619 iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2620                 uint32_t transport_fd, int is_leading)
2621 {
2622         struct iscsi_session *session = iscsi_ptr(sessionh);
2623         struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2624         struct sock *sk;
2625         struct socket *sock;
2626         int err;
2627
2628         /* lookup for existing socket */
2629         sock = sockfd_lookup(transport_fd, &err);
2630         if (!sock) {
2631                 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2632                 return -EEXIST;
2633         }
2634
2635         /* lookup for existing connection */
2636         spin_lock_bh(&session->lock);
2637         list_for_each_entry(tmp, &session->connections, item) {
2638                 if (tmp == conn) {
2639                         if (conn->c_stage != ISCSI_CONN_STOPPED ||
2640                             conn->stop_stage == STOP_CONN_TERM) {
2641                                 printk(KERN_ERR "iscsi_tcp: can't bind "
2642                                        "non-stopped connection (%d:%d)\n",
2643                                        conn->c_stage, conn->stop_stage);
2644                                 spin_unlock_bh(&session->lock);
2645                                 return -EIO;
2646                         }
2647                         break;
2648                 }
2649         }
2650         if (tmp != conn) {
2651                 /* bind new iSCSI connection to session */
2652                 conn->session = session;
2653
2654                 list_add(&conn->item, &session->connections);
2655         }
2656         spin_unlock_bh(&session->lock);
2657
2658         if (conn->stop_stage != STOP_CONN_SUSPEND) {
2659                 /* bind iSCSI connection and socket */
2660                 conn->sock = sock;
2661
2662                 /* setup Socket parameters */
2663                 sk = sock->sk;
2664                 sk->sk_reuse = 1;
2665                 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2666                 sk->sk_allocation = GFP_ATOMIC;
2667
2668                 /* FIXME: disable Nagle's algorithm */
2669
2670                 /*
2671                  * Intercept TCP callbacks for sendfile like receive
2672                  * processing.
2673                  */
2674                 iscsi_conn_set_callbacks(conn);
2675
2676                 /*
2677                  * set receive state machine into initial state
2678                  */
2679                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2680         }
2681
2682         if (is_leading)
2683                 session->leadconn = conn;
2684
2685         /*
2686          * Unblock xmitworker(), Login Phase will pass through.
2687          */
2688         clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2689         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2690
2691         return 0;
2692 }
2693
2694 static int
2695 iscsi_conn_start(iscsi_connh_t connh)
2696 {
2697         struct iscsi_conn *conn = iscsi_ptr(connh);
2698         struct iscsi_session *session = conn->session;
2699         struct sock *sk;
2700
2701         /* FF phase warming up... */
2702
2703         if (session == NULL) {
2704                 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2705                 return -EPERM;
2706         }
2707
2708         sk = conn->sock->sk;
2709
2710         write_lock_bh(&sk->sk_callback_lock);
2711         spin_lock_bh(&session->lock);
2712         conn->c_stage = ISCSI_CONN_STARTED;
2713         session->state = ISCSI_STATE_LOGGED_IN;
2714
2715         switch(conn->stop_stage) {
2716         case STOP_CONN_RECOVER:
2717                 /*
2718                  * unblock eh_abort() if it is blocked. re-try all
2719                  * commands after successful recovery
2720                  */
2721                 session->conn_cnt++;
2722                 conn->stop_stage = 0;
2723                 conn->tmabort_state = TMABORT_INITIAL;
2724                 session->age++;
2725                 wake_up(&conn->ehwait);
2726                 break;
2727         case STOP_CONN_TERM:
2728                 session->conn_cnt++;
2729                 conn->stop_stage = 0;
2730                 break;
2731         case STOP_CONN_SUSPEND:
2732                 conn->stop_stage = 0;
2733                 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2734                 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2735                 break;
2736         default:
2737                 break;
2738         }
2739         spin_unlock_bh(&session->lock);
2740         write_unlock_bh(&sk->sk_callback_lock);
2741
2742         return 0;
2743 }
2744
2745 static void
2746 iscsi_conn_stop(iscsi_connh_t connh, int flag)
2747 {
2748         struct iscsi_conn *conn = iscsi_ptr(connh);
2749         struct iscsi_session *session = conn->session;
2750         struct sock *sk;
2751         unsigned long flags;
2752
2753         BUG_ON(!conn->sock);
2754         sk = conn->sock->sk;
2755         write_lock_bh(&sk->sk_callback_lock);
2756         set_bit(SUSPEND_BIT, &conn->suspend_rx);
2757         write_unlock_bh(&sk->sk_callback_lock);
2758
2759         down(&conn->xmitsema);
2760
2761         spin_lock_irqsave(session->host->host_lock, flags);
2762         spin_lock(&session->lock);
2763         conn->stop_stage = flag;
2764         conn->c_stage = ISCSI_CONN_STOPPED;
2765         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2766
2767         if (flag != STOP_CONN_SUSPEND)
2768                 session->conn_cnt--;
2769
2770         if (session->conn_cnt == 0 || session->leadconn == conn)
2771                 session->state = ISCSI_STATE_FAILED;
2772
2773         spin_unlock(&session->lock);
2774         spin_unlock_irqrestore(session->host->host_lock, flags);
2775
2776         if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2777                 struct iscsi_cmd_task *ctask;
2778                 struct iscsi_mgmt_task *mtask;
2779
2780                 /*
2781                  * Socket must go now.
2782                  */
2783                 sock_hold(conn->sock->sk);
2784                 iscsi_conn_restore_callbacks(conn);
2785                 sock_put(conn->sock->sk);
2786
2787                 /*
2788                  * flush xmit queues.
2789                  */
2790                 spin_lock_bh(&session->lock);
2791                 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2792                             sizeof(void*)) ||
2793                         __kfifo_get(conn->xmitqueue, (void*)&ctask,
2794                             sizeof(void*))) {
2795                         struct iscsi_r2t_info *r2t;
2796
2797                         /*
2798                          * flush ctask's r2t queues
2799                          */
2800                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2801                                 sizeof(void*)))
2802                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2803                                             sizeof(void*));
2804
2805                         spin_unlock_bh(&session->lock);
2806                         local_bh_disable();
2807                         iscsi_ctask_cleanup(conn, ctask);
2808                         local_bh_enable();
2809                         spin_lock_bh(&session->lock);
2810                 }
2811                 conn->ctask = NULL;
2812                 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2813                            sizeof(void*)) ||
2814                         __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2815                            sizeof(void*))) {
2816                         __kfifo_put(session->mgmtpool.queue,
2817                                     (void*)&mtask, sizeof(void*));
2818                 }
2819                 conn->mtask = NULL;
2820                 spin_unlock_bh(&session->lock);
2821
2822                 /*
2823                  * release socket only after we stopped data_xmit()
2824                  * activity and flushed all outstandings
2825                  */
2826                 sock_release(conn->sock);
2827                 conn->sock = NULL;
2828
2829                 /*
2830                  * for connection level recovery we should not calculate
2831                  * header digest. conn->hdr_size used for optimization
2832                  * in hdr_extract() and will be re-negotiated at
2833                  * set_param() time.
2834                  */
2835                 if (flag == STOP_CONN_RECOVER)
2836                         conn->hdr_size = sizeof(struct iscsi_hdr);
2837         }
2838         up(&conn->xmitsema);
2839 }
2840
2841 static int
2842 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2843                         char *data, uint32_t data_size)
2844 {
2845         struct iscsi_session *session = conn->session;
2846         struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2847         struct iscsi_mgmt_task *mtask;
2848
2849         spin_lock_bh(&session->lock);
2850         if (session->state == ISCSI_STATE_TERMINATE) {
2851                 spin_unlock_bh(&session->lock);
2852                 return -EPERM;
2853         }
2854         if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2855             hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2856                 /*
2857                  * Login and Text are sent serially, in
2858                  * request-followed-by-response sequence.
2859                  * Same mtask can be used. Same ITT must be used.
2860                  * Note that login_mtask is preallocated at conn_create().
2861                  */
2862                 mtask = conn->login_mtask;
2863         else {
2864                 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2865                 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2866
2867                 if (!__kfifo_get(session->mgmtpool.queue,
2868                                  (void*)&mtask, sizeof(void*))) {
2869                         spin_unlock_bh(&session->lock);
2870                         return -ENOSPC;
2871                 }
2872         }
2873
2874         /*
2875          * pre-format CmdSN and ExpStatSN for outgoing PDU.
2876          */
2877         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2878                 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2879                            (session->age << AGE_SHIFT);
2880                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2881                 if (conn->c_stage == ISCSI_CONN_STARTED &&
2882                     !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2883                         session->cmdsn++;
2884         } else
2885                 /* do not advance CmdSN */
2886                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2887
2888         nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2889
2890         memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2891
2892         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE ||
2893             conn->stop_stage == STOP_CONN_RECOVER)
2894                 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2895                                     sizeof(struct iscsi_hdr));
2896         else
2897                 /* this will update header digest */
2898                 iscsi_buf_init_hdr(conn, &mtask->headbuf, (char*)&mtask->hdr,
2899                                     (u8 *)mtask->hdrext);
2900
2901         spin_unlock_bh(&session->lock);
2902
2903         if (data_size) {
2904                 memcpy(mtask->data, data, data_size);
2905                 mtask->data_count = data_size;
2906         } else
2907                 mtask->data_count = 0;
2908
2909         mtask->xmstate = XMSTATE_IMM_HDR;
2910
2911         if (mtask->data_count) {
2912                 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2913                                     mtask->data_count);
2914         }
2915
2916         debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2917                    hdr->opcode, hdr->itt, data_size);
2918
2919         /*
2920          * since send_pdu() could be called at least from two contexts,
2921          * we need to serialize __kfifo_put, so we don't have to take
2922          * additional lock on fast data-path
2923          */
2924         if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2925                 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2926         else
2927                 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2928
2929         schedule_work(&conn->xmitwork);
2930
2931         return 0;
2932 }
2933
2934 static int
2935 iscsi_eh_host_reset(struct scsi_cmnd *sc)
2936 {
2937         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2938         struct iscsi_conn *conn = ctask->conn;
2939         struct iscsi_session *session = conn->session;
2940
2941         spin_lock_bh(&session->lock);
2942         if (session->state == ISCSI_STATE_TERMINATE) {
2943                 debug_scsi("failing host reset: session terminated "
2944                            "[CID %d age %d]", conn->id, session->age);
2945                 spin_unlock_bh(&session->lock);
2946                 return FAILED;
2947         }
2948         spin_unlock_bh(&session->lock);
2949
2950         debug_scsi("failing connection CID %d due to SCSI host reset "
2951                    "[itt 0x%x age %d]", conn->id, ctask->itt,
2952                    session->age);
2953         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2954
2955         return SUCCESS;
2956 }
2957
2958 static void
2959 iscsi_tmabort_timedout(unsigned long data)
2960 {
2961         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2962         struct iscsi_conn *conn = ctask->conn;
2963         struct iscsi_session *session = conn->session;
2964
2965         spin_lock(&session->lock);
2966         if (conn->tmabort_state == TMABORT_INITIAL) {
2967                 __kfifo_put(session->mgmtpool.queue,
2968                                 (void*)&ctask->mtask, sizeof(void*));
2969                 conn->tmabort_state = TMABORT_TIMEDOUT;
2970                 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2971                         (long)ctask->sc, ctask->itt);
2972                 /* unblock eh_abort() */
2973                 wake_up(&conn->ehwait);
2974         }
2975         spin_unlock(&session->lock);
2976 }
2977
2978 static int
2979 iscsi_eh_abort(struct scsi_cmnd *sc)
2980 {
2981         int rc;
2982         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2983         struct iscsi_conn *conn = ctask->conn;
2984         struct iscsi_session *session = conn->session;
2985
2986         conn->eh_abort_cnt++;
2987         debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
2988
2989         /*
2990          * two cases for ERL=0 here:
2991          *
2992          * 1) connection-level failure;
2993          * 2) recovery due protocol error;
2994          */
2995         down(&conn->xmitsema);
2996         spin_lock_bh(&session->lock);
2997         if (session->state != ISCSI_STATE_LOGGED_IN) {
2998                 if (session->state == ISCSI_STATE_TERMINATE) {
2999                         spin_unlock_bh(&session->lock);
3000                         up(&conn->xmitsema);
3001                         goto failed;
3002                 }
3003                 spin_unlock_bh(&session->lock);
3004         } else {
3005                 struct iscsi_tm *hdr = &conn->tmhdr;
3006
3007                 /*
3008                  * Still LOGGED_IN...
3009                  */
3010
3011                 if (!ctask->sc || sc->SCp.phase != session->age) {
3012                         /*
3013                          * 1) ctask completed before time out. But session
3014                          *    is still ok => Happy Retry.
3015                          * 2) session was re-open during time out of ctask.
3016                          */
3017                         spin_unlock_bh(&session->lock);
3018                         up(&conn->xmitsema);
3019                         goto success;
3020                 }
3021                 conn->tmabort_state = TMABORT_INITIAL;
3022                 spin_unlock_bh(&session->lock);
3023
3024                 /*
3025                  * ctask timed out but session is OK
3026                  * ERL=0 requires task mgmt abort to be issued on each
3027                  * failed command. requests must be serialized.
3028                  */
3029                 memset(hdr, 0, sizeof(struct iscsi_tm));
3030                 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3031                 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3032                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3033                 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3034                 hdr->rtt = ctask->hdr.itt;
3035                 hdr->refcmdsn = ctask->hdr.cmdsn;
3036
3037                 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3038                                              NULL, 0);
3039                 if (rc) {
3040                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3041                         debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3042                 } else {
3043                         struct iscsi_r2t_info *r2t;
3044
3045                         /*
3046                          * TMF abort vs. TMF response race logic
3047                          */
3048                         spin_lock_bh(&session->lock);
3049                         ctask->mtask = (struct iscsi_mgmt_task *)
3050                                 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3051                                                         ISCSI_MGMT_ITT_OFFSET];
3052                         /*
3053                          * have to flush r2tqueue to avoid r2t leaks
3054                          */
3055                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3056                                 sizeof(void*))) {
3057                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3058                                         sizeof(void*));
3059                         }
3060                         if (conn->tmabort_state == TMABORT_INITIAL) {
3061                                 conn->tmfcmd_pdus_cnt++;
3062                                 conn->tmabort_timer.expires = 3*HZ + jiffies;
3063                                 conn->tmabort_timer.function =
3064                                                 iscsi_tmabort_timedout;
3065                                 conn->tmabort_timer.data = (unsigned long)ctask;
3066                                 add_timer(&conn->tmabort_timer);
3067                                 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3068                         } else {
3069                                 if (!ctask->sc ||
3070                                     conn->tmabort_state == TMABORT_SUCCESS) {
3071                                         conn->tmabort_state = TMABORT_INITIAL;
3072                                         spin_unlock_bh(&session->lock);
3073                                         up(&conn->xmitsema);
3074                                         goto success;
3075                                 }
3076                                 conn->tmabort_state = TMABORT_INITIAL;
3077                                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3078                         }
3079                         spin_unlock_bh(&session->lock);
3080                 }
3081         }
3082         up(&conn->xmitsema);
3083
3084
3085         /*
3086          * block eh thread until:
3087          *
3088          * 1) abort response;
3089          * 2) abort timeout;
3090          * 3) session re-opened;
3091          * 4) session terminated;
3092          */
3093         for (;;) {
3094                 int p_state = session->state;
3095
3096                 rc = wait_event_interruptible(conn->ehwait,
3097                         (p_state == ISCSI_STATE_LOGGED_IN ?
3098                          (session->state == ISCSI_STATE_TERMINATE ||
3099                           conn->tmabort_state != TMABORT_INITIAL) :
3100                          (session->state == ISCSI_STATE_TERMINATE ||
3101                           session->state == ISCSI_STATE_LOGGED_IN)));
3102                 if (rc) {
3103                         /* shutdown.. */
3104                         session->state = ISCSI_STATE_TERMINATE;
3105                         goto failed;
3106                 }
3107
3108                 if (signal_pending(current))
3109                         flush_signals(current);
3110
3111                 if (session->state == ISCSI_STATE_TERMINATE)
3112                         goto failed;
3113
3114                 spin_lock_bh(&session->lock);
3115                 if (sc->SCp.phase == session->age &&
3116                    (conn->tmabort_state == TMABORT_TIMEDOUT ||
3117                     conn->tmabort_state == TMABORT_FAILED)) {
3118                         conn->tmabort_state = TMABORT_INITIAL;
3119                         if (!ctask->sc) {
3120                                 /*
3121                                  * ctask completed before tmf abort response or
3122                                  * time out.
3123                                  * But session is still ok => Happy Retry.
3124                                  */
3125                                 spin_unlock_bh(&session->lock);
3126                                 break;
3127                         }
3128                         spin_unlock_bh(&session->lock);
3129                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3130                         continue;
3131                 }
3132                 spin_unlock_bh(&session->lock);
3133                 break;
3134         }
3135
3136 success:
3137         debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3138         rc = SUCCESS;
3139         goto exit;
3140
3141 failed:
3142         debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3143         rc = FAILED;
3144
3145 exit:
3146         del_timer_sync(&conn->tmabort_timer);
3147
3148         down(&conn->xmitsema);
3149         if (conn->sock) {
3150                 struct sock *sk = conn->sock->sk;
3151
3152                 write_lock_bh(&sk->sk_callback_lock);
3153                 iscsi_ctask_cleanup(conn, ctask);
3154                 write_unlock_bh(&sk->sk_callback_lock);
3155         }
3156         up(&conn->xmitsema);
3157         return rc;
3158 }
3159
3160 static int
3161 iscsi_r2tpool_alloc(struct iscsi_session *session)
3162 {
3163         int i;
3164         int cmd_i;
3165
3166         /*
3167          * initialize per-task: R2T pool and xmit queue
3168          */
3169         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3170                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3171
3172                 /*
3173                  * pre-allocated x4 as much r2ts to handle race when
3174                  * target acks DataOut faster than we data_xmit() queues
3175                  * could replenish r2tqueue.
3176                  */
3177
3178                 /* R2T pool */
3179                 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3180                         (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3181                         goto r2t_alloc_fail;
3182                 }
3183
3184                 /* R2T xmit queue */
3185                 ctask->r2tqueue = kfifo_alloc(
3186                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3187                 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3188                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);