drivers/scsi/iscsi_tcp.c

   1 /*
   2  * iSCSI Initiator over TCP/IP Data-Path
   3  *
   4  * Copyright (C) 2004 Dmitry Yusupov
   5  * Copyright (C) 2004 Alex Aizman
   6  * Copyright (C) 2005 Mike Christie
   7  * maintained by open-iscsi@googlegroups.com
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published
  11  * by the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17  * General Public License for more details.
  18  *
  19  * See the file COPYING included with this distribution for more details.
  20  *
  21  * Credits:
  22  *      Christoph Hellwig
  23  *      FUJITA Tomonori
  24  *      Arne Redlich
  25  *      Zhenyu Wang
  26  */
  27
  28 #include <linux/types.h>
  29 #include <linux/list.h>
  30 #include <linux/inet.h>
  31 #include <linux/blkdev.h>
  32 #include <linux/crypto.h>
  33 #include <linux/delay.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/scatterlist.h>
  36 #include <net/tcp.h>
  37 #include <scsi/scsi_cmnd.h>
  38 #include <scsi/scsi_device.h>
  39 #include <scsi/scsi_eh.h>
  40 #include <scsi/scsi_request.h>
  41 #include <scsi/scsi_tcq.h>
  42 #include <scsi/scsi_host.h>
  43 #include <scsi/scsi.h>
  44 #include <scsi/scsi_transport_iscsi.h>
  45
  46 #include "iscsi_tcp.h"
  47
  48 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  49               "Alex Aizman <itn780@yahoo.com>");
  50 MODULE_DESCRIPTION("iSCSI/TCP data-path");
  51 MODULE_LICENSE("GPL");
  52
  53 /* #define DEBUG_TCP */
  54 /* #define DEBUG_SCSI */
  55 #define DEBUG_ASSERT
  56
  57 #ifdef DEBUG_TCP
  58 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
  59 #else
  60 #define debug_tcp(fmt...)
  61 #endif
  62
  63 #ifdef DEBUG_SCSI
  64 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
  65 #else
  66 #define debug_scsi(fmt...)
  67 #endif
  68
  69 #ifndef DEBUG_ASSERT
  70 #ifdef BUG_ON
  71 #undef BUG_ON
  72 #endif
  73 #define BUG_ON(expr)
  74 #endif
  75
  76 #define INVALID_SN_DELTA        0xffff
  77
  78 static unsigned int iscsi_max_lun = 512;
  79 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  80
  81 /* global data */
  82 static kmem_cache_t *taskcache;
  83
  84 static inline void
  85 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
  86 {
  87         sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
  88         ibuf->sent = 0;
  89 }
  90
  91 static inline void
  92 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
  93 {
  94         ibuf->sg.page = (void*)vbuf;
  95         ibuf->sg.offset = (unsigned int)-1;
  96         ibuf->sg.length = size;
  97         ibuf->sent = 0;
  98 }
  99
 100 static inline void*
 101 iscsi_buf_iov_base(struct iscsi_buf *ibuf)
 102 {
 103         return (char*)ibuf->sg.page + ibuf->sent;
 104 }
 105
 106 static inline void
 107 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
 108 {
 109         /*
 110          * Fastpath: sg element fits into single page
 111          */
 112         if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) {
 113                 ibuf->sg.page = sg->page;
 114                 ibuf->sg.offset = sg->offset;
 115                 ibuf->sg.length = sg->length;
 116         } else
 117                 iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length);
 118         ibuf->sent = 0;
 119 }
 120
 121 static inline int
 122 iscsi_buf_left(struct iscsi_buf *ibuf)
 123 {
 124         int rc;
 125
 126         rc = ibuf->sg.length - ibuf->sent;
 127         BUG_ON(rc < 0);
 128         return rc;
 129 }
 130
 131 static inline void
 132 iscsi_buf_init_hdr(struct iscsi_conn *conn, struct iscsi_buf *ibuf,
 133                    char *vbuf, u8 *crc)
 134 {
 135         iscsi_buf_init_virt(ibuf, vbuf, sizeof(struct iscsi_hdr));
 136         if (conn->hdrdgst_en) {
 137                 crypto_digest_digest(conn->tx_tfm, &ibuf->sg, 1, crc);
 138                 ibuf->sg.length += sizeof(uint32_t);
 139         }
 140 }
 141
 142 static void
 143 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
 144 {
 145         struct iscsi_session *session = conn->session;
 146         unsigned long flags;
 147
 148         spin_lock_irqsave(&session->lock, flags);
 149         if (session->conn_cnt == 1 || session->leadconn == conn)
 150                 session->state = ISCSI_STATE_FAILED;
 151         spin_unlock_irqrestore(&session->lock, flags);
 152         set_bit(SUSPEND_BIT, &conn->suspend_tx);
 153         set_bit(SUSPEND_BIT, &conn->suspend_rx);
 154         iscsi_conn_error(iscsi_handle(conn), err);
 155 }
 156
 157 static inline int
 158 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
 159 {
 160         uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
 161         uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
 162
 163         if (max_cmdsn < exp_cmdsn -1 &&
 164             max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
 165                 return ISCSI_ERR_MAX_CMDSN;
 166         if (max_cmdsn > session->max_cmdsn ||
 167             max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
 168                 session->max_cmdsn = max_cmdsn;
 169         if (exp_cmdsn > session->exp_cmdsn ||
 170             exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
 171                 session->exp_cmdsn = exp_cmdsn;
 172
 173         return 0;
 174 }
 175
 176 static inline int
 177 iscsi_hdr_extract(struct iscsi_conn *conn)
 178 {
 179         struct sk_buff *skb = conn->in.skb;
 180
 181         if (conn->in.copy >= conn->hdr_size &&
 182             conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
 183                 /*
 184                  * Zero-copy PDU Header: using connection context
 185                  * to store header pointer.
 186                  */
 187                 if (skb_shinfo(skb)->frag_list == NULL &&
 188                     !skb_shinfo(skb)->nr_frags)
 189                         conn->in.hdr = (struct iscsi_hdr *)
 190                                 ((char*)skb->data + conn->in.offset);
 191                 else {
 192                         /* ignoring return code since we checked
 193                          * in.copy before */
 194                         skb_copy_bits(skb, conn->in.offset,
 195                                 &conn->hdr, conn->hdr_size);
 196                         conn->in.hdr = &conn->hdr;
 197                 }
 198                 conn->in.offset += conn->hdr_size;
 199                 conn->in.copy -= conn->hdr_size;
 200         } else {
 201                 int hdr_remains;
 202                 int copylen;
 203
 204                 /*
 205                  * PDU header scattered across SKB's,
 206                  * copying it... This'll happen quite rarely.
 207                  */
 208
 209                 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
 210                         conn->in.hdr_offset = 0;
 211
 212                 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
 213                 BUG_ON(hdr_remains <= 0);
 214
 215                 copylen = min(conn->in.copy, hdr_remains);
 216                 skb_copy_bits(skb, conn->in.offset,
 217                         (char*)&conn->hdr + conn->in.hdr_offset, copylen);
 218
 219                 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
 220                        "in.copy %d\n", conn->in.hdr_offset, copylen,
 221                        conn->in.offset, conn->in.copy);
 222
 223                 conn->in.offset += copylen;
 224                 conn->in.copy -= copylen;
 225                 if (copylen < hdr_remains)  {
 226                         conn->in_progress = IN_PROGRESS_HEADER_GATHER;
 227                         conn->in.hdr_offset += copylen;
 228                         return -EAGAIN;
 229                 }
 230                 conn->in.hdr = &conn->hdr;
 231                 conn->discontiguous_hdr_cnt++;
 232                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
 233         }
 234
 235         return 0;
 236 }
 237
 238 static inline void
 239 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 240 {
 241         struct scsi_cmnd *sc = ctask->sc;
 242         struct iscsi_session *session = conn->session;
 243
 244         spin_lock(&session->lock);
 245         if (unlikely(!sc)) {
 246                 spin_unlock(&session->lock);
 247                 return;
 248         }
 249         if (sc->sc_data_direction == DMA_TO_DEVICE) {
 250                 struct iscsi_data_task *dtask, *n;
 251                 /* WRITE: cleanup Data-Out's if any */
 252                 spin_lock(&conn->lock);
 253                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
 254                         list_del(&dtask->item);
 255                         mempool_free(dtask, ctask->datapool);
 256                 }
 257                 spin_unlock(&conn->lock);
 258         }
 259         ctask->xmstate = XMSTATE_IDLE;
 260         ctask->r2t = NULL;
 261         ctask->sc = NULL;
 262         __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
 263         spin_unlock(&session->lock);
 264 }
 265
 266 /**
 267  * iscsi_cmd_rsp - SCSI Command Response processing
 268  * @conn: iscsi connection
 269  * @ctask: scsi command task
 270  **/
 271 static int
 272 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 273 {
 274         int rc;
 275         struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
 276         struct iscsi_session *session = conn->session;
 277         struct scsi_cmnd *sc = ctask->sc;
 278
 279         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 280         if (rc) {
 281                 sc->result = (DID_ERROR << 16);
 282                 goto out;
 283         }
 284
 285         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 286
 287         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 288
 289         if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
 290                 sc->result = (DID_ERROR << 16);
 291                 goto out;
 292         }
 293
 294         if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
 295                 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
 296
 297                 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
 298                 debug_scsi("copied %d bytes of sense\n", sensecopy);
 299         }
 300
 301         if (sc->sc_data_direction == DMA_TO_DEVICE)
 302                 goto out;
 303
 304         if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 305                 int res_count = be32_to_cpu(rhdr->residual_count);
 306
 307                 if (res_count > 0 && res_count <= sc->request_bufflen)
 308                         sc->resid = res_count;
 309                 else
 310                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 311         } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
 312                 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 313         else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
 314                 sc->resid = be32_to_cpu(rhdr->residual_count);
 315
 316 out:
 317         debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 318                    (long)sc, sc->result, ctask->itt);
 319         conn->scsirsp_pdus_cnt++;
 320         iscsi_ctask_cleanup(conn, ctask);
 321         sc->scsi_done(sc);
 322         return rc;
 323 }
 324
 325 /**
 326  * iscsi_data_rsp - SCSI Data-In Response processing
 327  * @conn: iscsi connection
 328  * @ctask: scsi command task
 329  **/
 330 static int
 331 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 332 {
 333         int rc;
 334         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
 335         struct iscsi_session *session = conn->session;
 336         int datasn = be32_to_cpu(rhdr->datasn);
 337
 338         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 339         if (rc)
 340                 return rc;
 341         /*
 342          * setup Data-In byte counter (gets decremented..)
 343          */
 344         ctask->data_count = conn->in.datalen;
 345
 346         if (conn->in.datalen == 0)
 347                 return 0;
 348
 349         if (ctask->datasn != datasn)
 350                 return ISCSI_ERR_DATASN;
 351
 352         ctask->datasn++;
 353
 354         ctask->data_offset = be32_to_cpu(rhdr->offset);
 355         if (ctask->data_offset + conn->in.datalen > ctask->total_length)
 356                 return ISCSI_ERR_DATA_OFFSET;
 357
 358         if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
 359                 struct scsi_cmnd *sc = ctask->sc;
 360
 361                 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 362                 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 363                         int res_count = be32_to_cpu(rhdr->residual_count);
 364
 365                         if (res_count > 0 &&
 366                             res_count <= sc->request_bufflen) {
 367                                 sc->resid = res_count;
 368                                 sc->result = (DID_OK << 16) | rhdr->cmd_status;
 369                         } else
 370                                 sc->result = (DID_BAD_TARGET << 16) |
 371                                         rhdr->cmd_status;
 372                 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
 373                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 374                 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) {
 375                         sc->resid = be32_to_cpu(rhdr->residual_count);
 376                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 377                 } else
 378                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 379         }
 380
 381         conn->datain_pdus_cnt++;
 382         return 0;
 383 }
 384
 385 /**
 386  * iscsi_solicit_data_init - initialize first Data-Out
 387  * @conn: iscsi connection
 388  * @ctask: scsi command task
 389  * @r2t: R2T info
 390  *
 391  * Notes:
 392  *      Initialize first Data-Out within this R2T sequence and finds
 393  *      proper data_offset within this SCSI command.
 394  *
 395  *      This function is called with connection lock taken.
 396  **/
 397 static void
 398 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 399                         struct iscsi_r2t_info *r2t)
 400 {
 401         struct iscsi_data *hdr;
 402         struct iscsi_data_task *dtask;
 403         struct scsi_cmnd *sc = ctask->sc;
 404
 405         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
 406         BUG_ON(!dtask);
 407         hdr = &dtask->hdr;
 408         memset(hdr, 0, sizeof(struct iscsi_data));
 409         hdr->ttt = r2t->ttt;
 410         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
 411         r2t->solicit_datasn++;
 412         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
 413         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
 414         hdr->itt = ctask->hdr.itt;
 415         hdr->exp_statsn = r2t->exp_statsn;
 416         hdr->offset = cpu_to_be32(r2t->data_offset);
 417         if (r2t->data_length > conn->max_xmit_dlength) {
 418                 hton24(hdr->dlength, conn->max_xmit_dlength);
 419                 r2t->data_count = conn->max_xmit_dlength;
 420                 hdr->flags = 0;
 421         } else {
 422                 hton24(hdr->dlength, r2t->data_length);
 423                 r2t->data_count = r2t->data_length;
 424                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 425         }
 426         conn->dataout_pdus_cnt++;
 427
 428         r2t->sent = 0;
 429
 430         iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
 431                            (u8 *)dtask->hdrext);
 432
 433         r2t->dtask = dtask;
 434
 435         if (sc->use_sg) {
 436                 int i, sg_count = 0;
 437                 struct scatterlist *sg = sc->request_buffer;
 438
 439                 r2t->sg = NULL;
 440                 for (i = 0; i < sc->use_sg; i++, sg += 1) {
 441                         /* FIXME: prefetch ? */
 442                         if (sg_count + sg->length > r2t->data_offset) {
 443                                 int page_offset;
 444
 445                                 /* sg page found! */
 446
 447                                 /* offset within this page */
 448                                 page_offset = r2t->data_offset - sg_count;
 449
 450                                 /* fill in this buffer */
 451                                 iscsi_buf_init_sg(&r2t->sendbuf, sg);
 452                                 r2t->sendbuf.sg.offset += page_offset;
 453                                 r2t->sendbuf.sg.length -= page_offset;
 454
 455                                 /* xmit logic will continue with next one */
 456                                 r2t->sg = sg + 1;
 457                                 break;
 458                         }
 459                         sg_count += sg->length;
 460                 }
 461                 BUG_ON(r2t->sg == NULL);
 462         } else
 463                 iscsi_buf_init_iov(&ctask->sendbuf,
 464                             (char*)sc->request_buffer + r2t->data_offset,
 465                             r2t->data_count);
 466
 467         list_add(&dtask->item, &ctask->dataqueue);
 468 }
 469
 470 /**
 471  * iscsi_r2t_rsp - iSCSI R2T Response processing
 472  * @conn: iscsi connection
 473  * @ctask: scsi command task
 474  **/
 475 static int
 476 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 477 {
 478         struct iscsi_r2t_info *r2t;
 479         struct iscsi_session *session = conn->session;
 480         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
 481         int r2tsn = be32_to_cpu(rhdr->r2tsn);
 482         int rc;
 483
 484         if (conn->in.ahslen)
 485                 return ISCSI_ERR_AHSLEN;
 486
 487         if (conn->in.datalen)
 488                 return ISCSI_ERR_DATALEN;
 489
 490         if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
 491                 return ISCSI_ERR_R2TSN;
 492
 493         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 494         if (rc)
 495                 return rc;
 496
 497         /* FIXME: use R2TSN to detect missing R2T */
 498
 499         /* fill-in new R2T associated with the task */
 500         spin_lock(&session->lock);
 501         if (!ctask->sc || ctask->mtask ||
 502              session->state != ISCSI_STATE_LOGGED_IN) {
 503                 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
 504                        "recovery...\n", ctask->itt);
 505                 spin_unlock(&session->lock);
 506                 return 0;
 507         }
 508         rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
 509         BUG_ON(!rc);
 510
 511         r2t->exp_statsn = rhdr->statsn;
 512         r2t->data_length = be32_to_cpu(rhdr->data_length);
 513         if (r2t->data_length == 0 ||
 514             r2t->data_length > session->max_burst) {
 515                 spin_unlock(&session->lock);
 516                 return ISCSI_ERR_DATALEN;
 517         }
 518
 519         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
 520         if (r2t->data_offset + r2t->data_length > ctask->total_length) {
 521                 spin_unlock(&session->lock);
 522                 return ISCSI_ERR_DATALEN;
 523         }
 524
 525         r2t->ttt = rhdr->ttt; /* no flip */
 526         r2t->solicit_datasn = 0;
 527
 528         iscsi_solicit_data_init(conn, ctask, r2t);
 529
 530         ctask->exp_r2tsn = r2tsn + 1;
 531         ctask->xmstate |= XMSTATE_SOL_HDR;
 532         __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
 533         __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
 534
 535         schedule_work(&conn->xmitwork);
 536         conn->r2t_pdus_cnt++;
 537         spin_unlock(&session->lock);
 538
 539         return 0;
 540 }
 541
 542 static int
 543 iscsi_hdr_recv(struct iscsi_conn *conn)
 544 {
 545         int rc = 0;
 546         struct iscsi_hdr *hdr;
 547         struct iscsi_cmd_task *ctask;
 548         struct iscsi_session *session = conn->session;
 549         uint32_t cdgst, rdgst = 0;
 550
 551         hdr = conn->in.hdr;
 552
 553         /* verify PDU length */
 554         conn->in.datalen = ntoh24(hdr->dlength);
 555         if (conn->in.datalen > conn->max_recv_dlength) {
 556                 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
 557                        conn->in.datalen, conn->max_recv_dlength);
 558                 return ISCSI_ERR_DATALEN;
 559         }
 560         conn->data_copied = 0;
 561
 562         /* read AHS */
 563         conn->in.ahslen = hdr->hlength*(4*sizeof(__u16));
 564         conn->in.offset += conn->in.ahslen;
 565         conn->in.copy -= conn->in.ahslen;
 566         if (conn->in.copy < 0) {
 567                 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
 568                        "%d bytes\n", conn->in.ahslen);
 569                 return ISCSI_ERR_AHSLEN;
 570         }
 571
 572         /* calculate read padding */
 573         conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
 574         if (conn->in.padding) {
 575                 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
 576                 debug_scsi("read padding %d bytes\n", conn->in.padding);
 577         }
 578
 579         if (conn->hdrdgst_en) {
 580                 struct scatterlist sg;
 581
 582                 sg_init_one(&sg, (u8 *)hdr,
 583                             sizeof(struct iscsi_hdr) + conn->in.ahslen);
 584                 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
 585                 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
 586                                      conn->in.ahslen);
 587         }
 588
 589         /* save opcode for later */
 590         conn->in.opcode = hdr->opcode;
 591
 592         /* verify itt (itt encoding: age+cid+itt) */
 593         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
 594                 if ((hdr->itt & AGE_MASK) !=
 595                                 (session->age << AGE_SHIFT)) {
 596                         printk(KERN_ERR "iscsi_tcp: received itt %x expected "
 597                                 "session age (%x)\n", hdr->itt,
 598                                 session->age & AGE_MASK);
 599                         return ISCSI_ERR_BAD_ITT;
 600                 }
 601
 602                 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
 603                         printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
 604                                 "CID (%x)\n", hdr->itt, conn->id);
 605                         return ISCSI_ERR_BAD_ITT;
 606                 }
 607                 conn->in.itt = hdr->itt & ITT_MASK;
 608         } else
 609                 conn->in.itt = hdr->itt;
 610
 611         debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
 612                   hdr->opcode, conn->in.offset, conn->in.copy,
 613                   conn->in.ahslen, conn->in.datalen);
 614
 615         if (conn->in.itt < session->cmds_max) {
 616                 if (conn->hdrdgst_en && cdgst != rdgst) {
 617                         printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
 618                                "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
 619                                cdgst);
 620                         return ISCSI_ERR_HDR_DGST;
 621                 }
 622
 623                 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
 624
 625                 if (!ctask->sc) {
 626                         printk(KERN_INFO "iscsi_tcp: dropping ctask with "
 627                                "itt 0x%x\n", ctask->itt);
 628                         conn->in.datalen = 0; /* force drop */
 629                         return 0;
 630                 }
 631
 632                 if (ctask->sc->SCp.phase != session->age) {
 633                         printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
 634                                 "expected %d\n", ctask->sc->SCp.phase,
 635                                 session->age);
 636                         return ISCSI_ERR_SESSION_FAILED;
 637                 }
 638
 639                 conn->in.ctask = ctask;
 640
 641                 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
 642                            hdr->opcode, conn->id, (long)ctask->sc,
 643                            ctask->itt, conn->in.datalen);
 644
 645                 switch(conn->in.opcode) {
 646                 case ISCSI_OP_SCSI_CMD_RSP:
 647                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 648                         if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE)
 649                                 rc = iscsi_cmd_rsp(conn, ctask);
 650                         else if (!conn->in.datalen)
 651                                 rc = iscsi_cmd_rsp(conn, ctask);
 652                         else
 653                                 /*
 654                                  * got sense or response data; copying PDU
 655                                  * Header to the connection's header
 656                                  * placeholder
 657                                  */
 658                                 memcpy(&conn->hdr, hdr,
 659                                        sizeof(struct iscsi_hdr));
 660                         break;
 661                 case ISCSI_OP_SCSI_DATA_IN:
 662                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 663                         /* save flags for non-exceptional status */
 664                         conn->in.flags = hdr->flags;
 665                         /* save cmd_status for sense data */
 666                         conn->in.cmd_status =
 667                                 ((struct iscsi_data_rsp*)hdr)->cmd_status;
 668                         rc = iscsi_data_rsp(conn, ctask);
 669                         break;
 670                 case ISCSI_OP_R2T:
 671                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 672                         if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE &&
 673                             ctask->sc->sc_data_direction == DMA_TO_DEVICE)
 674                                 rc = iscsi_r2t_rsp(conn, ctask);
 675                         else
 676                                 rc = ISCSI_ERR_PROTO;
 677                         break;
 678                 case ISCSI_OP_NOOP_IN:
 679                 case ISCSI_OP_TEXT_RSP:
 680                 case ISCSI_OP_LOGOUT_RSP:
 681                 case ISCSI_OP_ASYNC_EVENT:
 682                 case ISCSI_OP_REJECT:
 683                         rc = iscsi_check_assign_cmdsn(session,
 684                                                  (struct iscsi_nopin*)hdr);
 685                         if (rc)
 686                                 break;
 687
 688                         /* update ExpStatSN */
 689                         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 690                         if (!conn->in.datalen) {
 691                                 struct iscsi_mgmt_task *mtask;
 692
 693                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 694                                                     NULL, 0);
 695                                 mtask = (struct iscsi_mgmt_task *)
 696                                         session->mgmt_cmds[conn->in.itt -
 697                                                 ISCSI_MGMT_ITT_OFFSET];
 698                                 if (conn->login_mtask != mtask) {
 699                                         spin_lock(&session->lock);
 700                                         __kfifo_put(session->mgmtpool.queue,
 701                                             (void*)&mtask, sizeof(void*));
 702                                         spin_unlock(&session->lock);
 703                                 }
 704                         }
 705                         break;
 706                 default:
 707                         rc = ISCSI_ERR_BAD_OPCODE;
 708                         break;
 709                 }
 710         } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
 711                    conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
 712                                         session->mgmtpool_max) {
 713                 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
 714                                         session->mgmt_cmds[conn->in.itt -
 715                                                 ISCSI_MGMT_ITT_OFFSET];
 716
 717                 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
 718                            conn->in.opcode, conn->id, mtask->itt,
 719                            conn->in.datalen);
 720
 721                 switch(conn->in.opcode) {
 722                 case ISCSI_OP_LOGIN_RSP:
 723                 case ISCSI_OP_TEXT_RSP:
 724                         rc = iscsi_check_assign_cmdsn(session,
 725                                                  (struct iscsi_nopin*)hdr);
 726                         if (rc)
 727                                 break;
 728
 729                         if (!conn->in.datalen) {
 730                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 731                                                     NULL, 0);
 732                                 if (conn->login_mtask != mtask) {
 733                                         spin_lock(&session->lock);
 734                                         __kfifo_put(session->mgmtpool.queue,
 735                                             (void*)&mtask, sizeof(void*));
 736                                         spin_unlock(&session->lock);
 737                                 }
 738                         }
 739                         break;
 740                 case ISCSI_OP_SCSI_TMFUNC_RSP:
 741                         rc = iscsi_check_assign_cmdsn(session,
 742                                                  (struct iscsi_nopin*)hdr);
 743                         if (rc)
 744                                 break;
 745
 746                         if (conn->in.datalen || conn->in.ahslen) {
 747                                 rc = ISCSI_ERR_PROTO;
 748                                 break;
 749                         }
 750                         conn->tmfrsp_pdus_cnt++;
 751                         spin_lock(&session->lock);
 752                         if (conn->tmabort_state == TMABORT_INITIAL) {
 753                                 __kfifo_put(session->mgmtpool.queue,
 754                                                 (void*)&mtask, sizeof(void*));
 755                                 conn->tmabort_state =
 756                                         ((struct iscsi_tm_rsp *)hdr)->
 757                                         response == SCSI_TCP_TM_RESP_COMPLETE ?
 758                                                 TMABORT_SUCCESS:TMABORT_FAILED;
 759                                 /* unblock eh_abort() */
 760                                 wake_up(&conn->ehwait);
 761                         }
 762                         spin_unlock(&session->lock);
 763                         break;
 764                 default:
 765                         rc = ISCSI_ERR_BAD_OPCODE;
 766                         break;
 767                 }
 768         } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
 769                 if (conn->in.opcode == ISCSI_OP_NOOP_IN && !conn->in.datalen) {
 770                         rc = iscsi_check_assign_cmdsn(session,
 771                                                  (struct iscsi_nopin*)hdr);
 772                         if (!rc)
 773                                 rc = iscsi_recv_pdu(iscsi_handle(conn),
 774                                                     hdr, NULL, 0);
 775                 }
 776                 else
 777                         rc = ISCSI_ERR_BAD_OPCODE;
 778         } else
 779                 rc = ISCSI_ERR_BAD_ITT;
 780
 781         return rc;
 782 }
 783
 784 /**
 785  * iscsi_ctask_copy - copy skb bits to the destanation cmd task
 786  * @conn: iscsi connection
 787  * @ctask: scsi command task
 788  * @buf: buffer to copy to
 789  * @buf_size: size of buffer
 790  * @offset: offset within the buffer
 791  *
 792  * Notes:
 793  *      The function calls skb_copy_bits() and updates per-connection and
 794  *      per-cmd byte counters.
 795  *
 796  *      Read counters (in bytes):
 797  *
 798  *      conn->in.offset         offset within in progress SKB
 799  *      conn->in.copy           left to copy from in progress SKB
 800  *                              including padding
 801  *      conn->in.copied         copied already from in progress SKB
 802  *      conn->data_copied       copied already from in progress buffer
 803  *      ctask->sent             total bytes sent up to the MidLayer
 804  *      ctask->data_count       left to copy from in progress Data-In
 805  *      buf_left                left to copy from in progress buffer
 806  **/
 807 static inline int
 808 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 809                 void *buf, int buf_size, int offset)
 810 {
 811         int buf_left = buf_size - (conn->data_copied + offset);
 812         int size = min(conn->in.copy, buf_left);
 813         int rc;
 814
 815         size = min(size, ctask->data_count);
 816
 817         debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
 818                size, conn->in.offset, conn->in.copied);
 819
 820         BUG_ON(size <= 0);
 821         BUG_ON(ctask->sent + size > ctask->total_length);
 822
 823         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 824                            (char*)buf + (offset + conn->data_copied), size);
 825         /* must fit into skb->len */
 826         BUG_ON(rc);
 827
 828         conn->in.offset += size;
 829         conn->in.copy -= size;
 830         conn->in.copied += size;
 831         conn->data_copied += size;
 832         ctask->sent += size;
 833         ctask->data_count -= size;
 834
 835         BUG_ON(conn->in.copy < 0);
 836         BUG_ON(ctask->data_count < 0);
 837
 838         if (buf_size != (conn->data_copied + offset)) {
 839                 if (!ctask->data_count) {
 840                         BUG_ON(buf_size - conn->data_copied < 0);
 841                         /* done with this PDU */
 842                         return buf_size - conn->data_copied;
 843                 }
 844                 return -EAGAIN;
 845         }
 846
 847         /* done with this buffer or with both - PDU and buffer */
 848         conn->data_copied = 0;
 849         return 0;
 850 }
 851
 852 /**
 853  * iscsi_tcp_copy - copy skb bits to the destanation buffer
 854  * @conn: iscsi connection
 855  * @buf: buffer to copy to
 856  * @buf_size: number of bytes to copy
 857  *
 858  * Notes:
 859  *      The function calls skb_copy_bits() and updates per-connection
 860  *      byte counters.
 861  **/
 862 static inline int
 863 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
 864 {
 865         int buf_left = buf_size - conn->data_copied;
 866         int size = min(conn->in.copy, buf_left);
 867         int rc;
 868
 869         debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
 870                size, conn->in.offset, conn->data_copied);
 871         BUG_ON(size <= 0);
 872
 873         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 874                            (char*)buf + conn->data_copied, size);
 875         BUG_ON(rc);
 876
 877         conn->in.offset += size;
 878         conn->in.copy -= size;
 879         conn->in.copied += size;
 880         conn->data_copied += size;
 881
 882         if (buf_size != conn->data_copied)
 883                 return -EAGAIN;
 884
 885         return 0;
 886 }
 887
 888 static inline void
 889 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
 890                          int offset, int length)
 891 {
 892         struct scatterlist temp;
 893
 894         memcpy(&temp, sg, sizeof(struct scatterlist));
 895         temp.offset = offset;
 896         temp.length = length;
 897         crypto_digest_update(conn->data_rx_tfm, &temp, 1);
 898 }
 899
 900 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 901 {
 902         struct iscsi_cmd_task *ctask = conn->in.ctask;
 903         struct scsi_cmnd *sc = ctask->sc;
 904         struct scatterlist tmp, *sg;
 905         int i, offset, rc = 0;
 906
 907         BUG_ON((void*)ctask != sc->SCp.ptr);
 908
 909         /*
 910          * copying Data-In into the Scsi_Cmnd
 911          */
 912         if (!sc->use_sg) {
 913                 i = ctask->data_count;
 914                 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
 915                                       sc->request_bufflen, ctask->data_offset);
 916                 if (rc == -EAGAIN)
 917                         return rc;
 918                 if (conn->datadgst_en) {
 919                         sg_init_one(&tmp, sc->request_buffer, i);
 920                         crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
 921                 }
 922                 rc = 0;
 923                 goto done;
 924         }
 925
 926         offset = ctask->data_offset;
 927         sg = sc->request_buffer;
 928
 929         if (ctask->data_offset)
 930                 for (i = 0; i < ctask->sg_count; i++)
 931                         offset -= sg[i].length;
 932         /* we've passed through partial sg*/
 933         if (offset < 0)
 934                 offset = 0;
 935
 936         for (i = ctask->sg_count; i < sc->use_sg; i++) {
 937                 char *dest;
 938
 939                 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
 940                 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
 941                                       sg[i].length, offset);
 942                 kunmap_atomic(dest, KM_SOFTIRQ0);
 943                 if (rc == -EAGAIN)
 944                         /* continue with the next SKB/PDU */
 945                         return rc;
 946                 if (!rc) {
 947                         if (conn->datadgst_en) {
 948                                 if (!offset)
 949                                         crypto_digest_update(conn->data_rx_tfm,
 950                                                              &sg[i], 1);
 951                                 else
 952                                         partial_sg_digest_update(conn, &sg[i],
 953                                                         sg[i].offset + offset,
 954                                                         sg[i].length - offset);
 955                         }
 956                         offset = 0;
 957                         ctask->sg_count++;
 958                 }
 959
 960                 if (!ctask->data_count) {
 961                         if (rc && conn->datadgst_en)
 962                                 /*
 963                                  * data-in is complete, but buffer not...
 964                                  */
 965                                 partial_sg_digest_update(conn, &sg[i],
 966                                                 sg[i].offset, sg[i].length-rc);
 967                         rc = 0;
 968                         break;
 969                 }
 970
 971                 if (!conn->in.copy)
 972                         return -EAGAIN;
 973         }
 974         BUG_ON(ctask->data_count);
 975
 976 done:
 977         /* check for non-exceptional status */
 978         if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
 979                 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 980                            (long)sc, sc->result, ctask->itt);
 981                 conn->scsirsp_pdus_cnt++;
 982                 iscsi_ctask_cleanup(conn, ctask);
 983                 sc->scsi_done(sc);
 984         }
 985
 986         return rc;
 987 }
 988
 989 static int
 990 iscsi_data_recv(struct iscsi_conn *conn)
 991 {
 992         struct iscsi_session *session = conn->session;
 993         int rc = 0;
 994
 995         switch(conn->in.opcode) {
 996         case ISCSI_OP_SCSI_DATA_IN:
 997                 rc = iscsi_scsi_data_in(conn);
 998                 break;
 999         case ISCSI_OP_SCSI_CMD_RSP: {
1000                 /*
1001                  * SCSI Sense Data:
1002                  * copying the entire Data Segment.
1003                  */
1004                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1005                         rc = -EAGAIN;
1006                         goto exit;
1007                 }
1008
1009                 /*
1010                  * check for sense
1011                  */
1012                 conn->in.hdr = &conn->hdr;
1013                 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1014                 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1015         }
1016         break;
1017         case ISCSI_OP_TEXT_RSP:
1018         case ISCSI_OP_LOGIN_RSP:
1019         case ISCSI_OP_NOOP_IN: {
1020                 struct iscsi_mgmt_task *mtask = NULL;
1021
1022                 if (conn->in.itt != ISCSI_RESERVED_TAG)
1023                         mtask = (struct iscsi_mgmt_task *)
1024                                 session->mgmt_cmds[conn->in.itt -
1025                                         ISCSI_MGMT_ITT_OFFSET];
1026
1027                 /*
1028                  * Collect data segment to the connection's data
1029                  * placeholder
1030                  */
1031                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1032                         rc = -EAGAIN;
1033                         goto exit;
1034                 }
1035
1036                 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1037                                     conn->data, conn->in.datalen);
1038
1039                 if (mtask && conn->login_mtask != mtask) {
1040                         spin_lock(&session->lock);
1041                         __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1042                                     sizeof(void*));
1043                         spin_unlock(&session->lock);
1044                 }
1045         }
1046         break;
1047         default:
1048                 BUG_ON(1);
1049         }
1050 exit:
1051         return rc;
1052 }
1053
1054 /**
1055  * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1056  * @rd_desc: read descriptor
1057  * @skb: socket buffer
1058  * @offset: offset in skb
1059  * @len: skb->len - offset
1060  **/
1061 static int
1062 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1063                 unsigned int offset, size_t len)
1064 {
1065         int rc;
1066         struct iscsi_conn *conn = rd_desc->arg.data;
1067         int start = skb_headlen(skb);
1068         int processed;
1069         char pad[ISCSI_PAD_LEN];
1070         struct scatterlist sg;
1071
1072         /*
1073          * Save current SKB and its offset in the corresponding
1074          * connection context.
1075          */
1076         conn->in.copy = start - offset;
1077         conn->in.offset = offset;
1078         conn->in.skb = skb;
1079         conn->in.len = conn->in.copy;
1080         BUG_ON(conn->in.copy <= 0);
1081         debug_tcp("in %d bytes\n", conn->in.copy);
1082
1083 more:
1084         conn->in.copied = 0;
1085         rc = 0;
1086
1087         if (unlikely(conn->suspend_rx)) {
1088                 debug_tcp("conn %d Rx suspended!\n", conn->id);
1089                 return 0;
1090         }
1091
1092         if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1093             conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1094                 rc = iscsi_hdr_extract(conn);
1095                 if (rc) {
1096                        if (rc == -EAGAIN)
1097                                 goto nomore;
1098                        else {
1099                                 iscsi_conn_failure(conn, rc);
1100                                 return 0;
1101                        }
1102                 }
1103
1104                 /*
1105                  * Verify and process incoming PDU header.
1106                  */
1107                 rc = iscsi_hdr_recv(conn);
1108                 if (!rc && conn->in.datalen) {
1109                         if (conn->datadgst_en &&
1110                             conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1111                                 BUG_ON(!conn->data_rx_tfm);
1112                                 crypto_digest_init(conn->data_rx_tfm);
1113                         }
1114                         conn->in_progress = IN_PROGRESS_DATA_RECV;
1115                 } else if (rc) {
1116                         iscsi_conn_failure(conn, rc);
1117                         return 0;
1118                 }
1119         }
1120
1121         if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1122                 debug_tcp("extra data_recv offset %d copy %d\n",
1123                           conn->in.offset, conn->in.copy);
1124                 if (conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1125                         uint32_t recv_digest;
1126                         skb_copy_bits(conn->in.skb, conn->in.offset,
1127                                       &recv_digest, 4);
1128                         conn->in.offset += 4;
1129                         conn->in.copy -= 4;
1130                         if (recv_digest != conn->in.datadgst) {
1131                                 debug_tcp("iscsi_tcp: data digest error!"
1132                                           "0x%x != 0x%x\n", recv_digest,
1133                                           conn->in.datadgst);
1134                                 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1135                                 return 0;
1136                         } else {
1137                                 debug_tcp("iscsi_tcp: data digest match!"
1138                                           "0x%x == 0x%x\n", recv_digest,
1139                                           conn->in.datadgst);
1140                                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1141                         }
1142                 }
1143         }
1144
1145         if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1146
1147                 debug_tcp("data_recv offset %d copy %d\n",
1148                        conn->in.offset, conn->in.copy);
1149
1150                 rc = iscsi_data_recv(conn);
1151                 if (rc) {
1152                         if (rc == -EAGAIN) {
1153                                 rd_desc->count = conn->in.datalen -
1154                                                 conn->in.ctask->data_count;
1155                                 goto again;
1156                         }
1157                         iscsi_conn_failure(conn, rc);
1158                         return 0;
1159                 }
1160                 conn->in.copy -= conn->in.padding;
1161                 conn->in.offset += conn->in.padding;
1162                 if (conn->datadgst_en &&
1163                     conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
1164                         if (conn->in.padding) {
1165                                 debug_tcp("padding -> %d\n", conn->in.padding);
1166                                 memset(pad, 0, conn->in.padding);
1167                                 sg_init_one(&sg, pad, conn->in.padding);
1168                                 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1169                         }
1170                         crypto_digest_final(conn->data_rx_tfm,
1171                                             (u8 *) & conn->in.datadgst);
1172                         debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1173                         conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1174                 } else
1175                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1176         }
1177
1178         debug_tcp("f, processed %d from out of %d padding %d\n",
1179                conn->in.offset - offset, (int)len, conn->in.padding);
1180         BUG_ON(conn->in.offset - offset > len);
1181
1182         if (conn->in.offset - offset != len) {
1183                 debug_tcp("continue to process %d bytes\n",
1184                        (int)len - (conn->in.offset - offset));
1185                 goto more;
1186         }
1187
1188 nomore:
1189         processed = conn->in.offset - offset;
1190         BUG_ON(processed == 0);
1191         return processed;
1192
1193 again:
1194         processed = conn->in.offset - offset;
1195         debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1196                   processed, (int)len, (int)rd_desc->count);
1197         BUG_ON(processed == 0);
1198         BUG_ON(processed > len);
1199
1200         conn->rxdata_octets += processed;
1201         return processed;
1202 }
1203
1204 static void
1205 iscsi_tcp_data_ready(struct sock *sk, int flag)
1206 {
1207         struct iscsi_conn *conn = sk->sk_user_data;
1208         read_descriptor_t rd_desc;
1209
1210         read_lock(&sk->sk_callback_lock);
1211
1212         /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1213         rd_desc.arg.data = conn;
1214         rd_desc.count = 0;
1215         tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1216
1217         read_unlock(&sk->sk_callback_lock);
1218 }
1219
1220 static void
1221 iscsi_tcp_state_change(struct sock *sk)
1222 {
1223         struct iscsi_conn *conn;
1224         struct iscsi_session *session;
1225         void (*old_state_change)(struct sock *);
1226
1227         read_lock(&sk->sk_callback_lock);
1228
1229         conn = (struct iscsi_conn*)sk->sk_user_data;
1230         session = conn->session;
1231
1232         if (sk->sk_state == TCP_CLOSE_WAIT ||
1233             sk->sk_state == TCP_CLOSE) {
1234                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1235                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1236         }
1237
1238         old_state_change = conn->old_state_change;
1239
1240         read_unlock(&sk->sk_callback_lock);
1241
1242         old_state_change(sk);
1243 }
1244
1245 /**
1246  * iscsi_write_space - Called when more output buffer space is available
1247  * @sk: socket space is available for
1248  **/
1249 static void
1250 iscsi_write_space(struct sock *sk)
1251 {
1252         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1253         conn->old_write_space(sk);
1254         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1255         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1256         schedule_work(&conn->xmitwork);
1257 }
1258
1259 static void
1260 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1261 {
1262         struct sock *sk = conn->sock->sk;
1263
1264         /* assign new callbacks */
1265         write_lock_bh(&sk->sk_callback_lock);
1266         sk->sk_user_data = conn;
1267         conn->old_data_ready = sk->sk_data_ready;
1268         conn->old_state_change = sk->sk_state_change;
1269         conn->old_write_space = sk->sk_write_space;
1270         sk->sk_data_ready = iscsi_tcp_data_ready;
1271         sk->sk_state_change = iscsi_tcp_state_change;
1272         sk->sk_write_space = iscsi_write_space;
1273         write_unlock_bh(&sk->sk_callback_lock);
1274 }
1275
1276 static void
1277 iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1278 {
1279         struct sock *sk = conn->sock->sk;
1280
1281         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1282         write_lock_bh(&sk->sk_callback_lock);
1283         sk->sk_user_data    = NULL;
1284         sk->sk_data_ready   = conn->old_data_ready;
1285         sk->sk_state_change = conn->old_state_change;
1286         sk->sk_write_space  = conn->old_write_space;
1287         sk->sk_no_check  = 0;
1288         write_unlock_bh(&sk->sk_callback_lock);
1289 }
1290
1291 /**
1292  * iscsi_send - generic send routine
1293  * @sk: kernel's socket
1294  * @buf: buffer to write from
1295  * @size: actual size to write
1296  * @flags: socket's flags
1297  *
1298  * Notes:
1299  *      depending on buffer will use tcp_sendpage() or tcp_sendmsg().
1300  *      buf->sg.offset == -1 tells us that buffer is non S/G and forces
1301  *      to use tcp_sendmsg().
1302  */
1303 static inline int
1304 iscsi_send(struct socket *sk, struct iscsi_buf *buf, int size, int flags)
1305 {
1306         int res;
1307
1308         if ((int)buf->sg.offset >= 0) {
1309                 int offset = buf->sg.offset + buf->sent;
1310
1311                 /* tcp_sendpage */
1312                 res = sk->ops->sendpage(sk, buf->sg.page, offset, size, flags);
1313         } else {
1314                 struct msghdr msg;
1315
1316                 buf->iov.iov_base = iscsi_buf_iov_base(buf);
1317                 buf->iov.iov_len = size;
1318
1319                 memset(&msg, 0, sizeof(struct msghdr));
1320
1321                 /* tcp_sendmsg */
1322                 res = kernel_sendmsg(sk, &msg, &buf->iov, 1, size);
1323         }
1324
1325         return res;
1326 }
1327
1328 /**
1329  * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1330  * @conn: iscsi connection
1331  * @buf: buffer to write from
1332  * @datalen: lenght of data to be sent after the header
1333  *
1334  * Notes:
1335  *      (Tx, Fast Path)
1336  **/
1337 static inline int
1338 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1339 {
1340         struct socket *sk = conn->sock;
1341         int flags = 0; /* MSG_DONTWAIT; */
1342         int res, size;
1343
1344         size = buf->sg.length - buf->sent;
1345         BUG_ON(buf->sent + size > buf->sg.length);
1346         if (buf->sent + size != buf->sg.length || datalen)
1347                 flags |= MSG_MORE;
1348
1349         res = iscsi_send(sk, buf, size, flags);
1350         debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1351         if (res >= 0) {
1352                 conn->txdata_octets += res;
1353                 buf->sent += res;
1354                 if (size != res)
1355                         return -EAGAIN;
1356                 return 0;
1357         } else if (res == -EAGAIN) {
1358                 conn->sendpage_failures_cnt++;
1359                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1360         } else if (res == -EPIPE)
1361                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1362
1363         return res;
1364 }
1365
1366 /**
1367  * iscsi_sendpage - send one page of iSCSI Data-Out.
1368  * @conn: iscsi connection
1369  * @buf: buffer to write from
1370  * @count: remaining data
1371  * @sent: number of bytes sent
1372  *
1373  * Notes:
1374  *      (Tx, Fast Path)
1375  **/
1376 static inline int
1377 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1378                int *count, int *sent)
1379 {
1380         struct socket *sk = conn->sock;
1381         int flags = 0; /* MSG_DONTWAIT; */
1382         int res, size;
1383
1384         size = buf->sg.length - buf->sent;
1385         BUG_ON(buf->sent + size > buf->sg.length);
1386         if (size > *count)
1387                 size = *count;
1388         if (buf->sent + size != buf->sg.length)
1389                 flags |= MSG_MORE;
1390
1391         res = iscsi_send(sk, buf, size, flags);
1392         debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1393                   size, buf->sent, *count, *sent, res);
1394         if (res >= 0) {
1395                 conn->txdata_octets += res;
1396                 buf->sent += res;
1397                 *count -= res;
1398                 *sent += res;
1399                 if (size != res)
1400                         return -EAGAIN;
1401                 return 0;
1402         } else if (res == -EAGAIN) {
1403                 conn->sendpage_failures_cnt++;
1404                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1405         } else if (res == -EPIPE)
1406                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1407
1408         return res;
1409 }
1410
1411 static inline void
1412 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1413 {
1414         BUG_ON(!conn->data_tx_tfm);
1415         crypto_digest_init(conn->data_tx_tfm);
1416         ctask->digest_count = 4;
1417 }
1418
1419 static inline void
1420 iscsi_buf_data_digest_update(struct iscsi_conn *conn, struct iscsi_buf *buf)
1421 {
1422         struct scatterlist sg;
1423
1424         if (buf->sg.offset != -1)
1425                 crypto_digest_update(conn->data_tx_tfm, &buf->sg, 1);
1426         else {
1427                 sg_init_one(&sg, (char *)buf->sg.page, buf->sg.length);
1428                 crypto_digest_update(conn->data_tx_tfm, &sg, 1);
1429         }
1430 }
1431
1432 static inline int
1433 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1434                         struct iscsi_buf *buf, uint32_t *digest, int final)
1435 {
1436         int rc = 0;
1437         int sent = 0;
1438
1439         if (final)
1440                 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1441
1442         iscsi_buf_init_virt(buf, (char*)digest, 4);
1443         rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1444         if (rc) {
1445                 ctask->datadigest = *digest;
1446                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1447         } else
1448                 ctask->digest_count = 4;
1449         return rc;
1450 }
1451
1452 /**
1453  * iscsi_solicit_data_cont - initialize next Data-Out
1454  * @conn: iscsi connection
1455  * @ctask: scsi command task
1456  * @r2t: R2T info
1457  * @left: bytes left to transfer
1458  *
1459  * Notes:
1460  *      Initialize next Data-Out within this R2T sequence and continue
1461  *      to process next Scatter-Gather element(if any) of this SCSI command.
1462  *
1463  *      Called under connection lock.
1464  **/
1465 static void
1466 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1467                         struct iscsi_r2t_info *r2t, int left)
1468 {
1469         struct iscsi_data *hdr;
1470         struct iscsi_data_task *dtask;
1471         struct scsi_cmnd *sc = ctask->sc;
1472         int new_offset;
1473
1474         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1475         BUG_ON(!dtask);
1476         hdr = &dtask->hdr;
1477         memset(hdr, 0, sizeof(struct iscsi_data));
1478         hdr->ttt = r2t->ttt;
1479         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1480         r2t->solicit_datasn++;
1481         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1482         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1483         hdr->itt = ctask->hdr.itt;
1484         hdr->exp_statsn = r2t->exp_statsn;
1485         new_offset = r2t->data_offset + r2t->sent;
1486         hdr->offset = cpu_to_be32(new_offset);
1487         if (left > conn->max_xmit_dlength) {
1488                 hton24(hdr->dlength, conn->max_xmit_dlength);
1489                 r2t->data_count = conn->max_xmit_dlength;
1490         } else {
1491                 hton24(hdr->dlength, left);
1492                 r2t->data_count = left;
1493                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1494         }
1495         conn->dataout_pdus_cnt++;
1496
1497         iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
1498                            (u8 *)dtask->hdrext);
1499
1500         r2t->dtask = dtask;
1501
1502         if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1503                 BUG_ON(ctask->bad_sg == r2t->sg);
1504                 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1505                 r2t->sg += 1;
1506         } else
1507                 iscsi_buf_init_iov(&ctask->sendbuf,
1508                             (char*)sc->request_buffer + new_offset,
1509                             r2t->data_count);
1510
1511         list_add(&dtask->item, &ctask->dataqueue);
1512 }
1513
1514 static void
1515 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1516 {
1517         struct iscsi_data *hdr;
1518         struct iscsi_data_task *dtask;
1519
1520         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1521         BUG_ON(!dtask);
1522         hdr = &dtask->hdr;
1523         memset(hdr, 0, sizeof(struct iscsi_data));
1524         hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1525         hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1526         ctask->unsol_datasn++;
1527         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1528         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1529         hdr->itt = ctask->hdr.itt;
1530         hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1531         hdr->offset = cpu_to_be32(ctask->total_length -
1532                                   ctask->r2t_data_count -
1533                                   ctask->unsol_count);
1534         if (ctask->unsol_count > conn->max_xmit_dlength) {
1535                 hton24(hdr->dlength, conn->max_xmit_dlength);
1536                 ctask->data_count = conn->max_xmit_dlength;
1537                 hdr->flags = 0;
1538         } else {
1539                 hton24(hdr->dlength, ctask->unsol_count);
1540                 ctask->data_count = ctask->unsol_count;
1541                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1542         }
1543
1544         iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)hdr,
1545                            (u8 *)dtask->hdrext);
1546
1547         list_add(&dtask->item, &ctask->dataqueue);
1548
1549         ctask->dtask = dtask;
1550 }
1551
1552 /**
1553  * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1554  * @conn: iscsi connection
1555  * @ctask: scsi command task
1556  * @sc: scsi command
1557  **/
1558 static void
1559 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1560                 struct scsi_cmnd *sc)
1561 {
1562         struct iscsi_session *session = conn->session;
1563
1564         BUG_ON(__kfifo_len(ctask->r2tqueue));
1565
1566         ctask->sc = sc;
1567         ctask->conn = conn;
1568         ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1569         ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1570         int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1571         ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1572                          (session->age << AGE_SHIFT);
1573         ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1574         ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1575         ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1576         memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1577         memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1578
1579         ctask->mtask = NULL;
1580         ctask->sent = 0;
1581         ctask->sg_count = 0;
1582
1583         ctask->total_length = sc->request_bufflen;
1584
1585         if (sc->sc_data_direction == DMA_TO_DEVICE) {
1586                 ctask->exp_r2tsn = 0;
1587                 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1588                 BUG_ON(ctask->total_length == 0);
1589                 if (sc->use_sg) {
1590                         struct scatterlist *sg = sc->request_buffer;
1591
1592                         iscsi_buf_init_sg(&ctask->sendbuf,
1593                                           &sg[ctask->sg_count++]);
1594                         ctask->sg = sg;
1595                         ctask->bad_sg = sg + sc->use_sg;
1596                 } else {
1597                         iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1598                                         sc->request_bufflen);
1599                 }
1600
1601                 /*
1602                  * Write counters:
1603                  *
1604                  *      imm_count       bytes to be sent right after
1605                  *                      SCSI PDU Header
1606                  *
1607                  *      unsol_count     bytes(as Data-Out) to be sent
1608                  *                      without R2T ack right after
1609                  *                      immediate data
1610                  *
1611                  *      r2t_data_count  bytes to be sent via R2T ack's
1612                  *
1613                  *      pad_count       bytes to be sent as zero-padding
1614                  */
1615                 ctask->imm_count = 0;
1616                 ctask->unsol_count = 0;
1617                 ctask->unsol_datasn = 0;
1618                 ctask->xmstate = XMSTATE_W_HDR;
1619                 /* calculate write padding */
1620                 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1621                 if (ctask->pad_count) {
1622                         ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1623                         debug_scsi("write padding %d bytes\n",
1624                                 ctask->pad_count);
1625                         ctask->xmstate |= XMSTATE_W_PAD;
1626                 }
1627                 if (session->imm_data_en) {
1628                         if (ctask->total_length >= session->first_burst)
1629                                 ctask->imm_count = min(session->first_burst,
1630                                                         conn->max_xmit_dlength);
1631                         else
1632                                 ctask->imm_count = min(ctask->total_length,
1633                                                         conn->max_xmit_dlength);
1634                         hton24(ctask->hdr.dlength, ctask->imm_count);
1635                         ctask->xmstate |= XMSTATE_IMM_DATA;
1636                 } else
1637                         zero_data(ctask->hdr.dlength);
1638
1639                 if (!session->initial_r2t_en)
1640                         ctask->unsol_count = min(session->first_burst,
1641                                 ctask->total_length) - ctask->imm_count;
1642                 if (!ctask->unsol_count)
1643                         /* No unsolicit Data-Out's */
1644                         ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1645                 else
1646                         ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1647
1648                 ctask->r2t_data_count = ctask->total_length -
1649                                     ctask->imm_count -
1650                                     ctask->unsol_count;
1651
1652                 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1653                            "r2t_data %d]\n",
1654                            ctask->itt, ctask->total_length, ctask->imm_count,
1655                            ctask->unsol_count, ctask->r2t_data_count);
1656         } else {
1657                 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1658                 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1659                         ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1660                 ctask->datasn = 0;
1661                 ctask->xmstate = XMSTATE_R_HDR;
1662                 zero_data(ctask->hdr.dlength);
1663         }
1664
1665         iscsi_buf_init_hdr(conn, &ctask->headbuf, (char*)&ctask->hdr,
1666                             (u8 *)ctask->hdrext);
1667         conn->scsicmd_pdus_cnt++;
1668 }
1669
1670 /**
1671  * iscsi_mtask_xmit - xmit management(immediate) task
1672  * @conn: iscsi connection
1673  * @mtask: task management task
1674  *
1675  * Notes:
1676  *      The function can return -EAGAIN in which case caller must
1677  *      call it again later, or recover. '0' return code means successful
1678  *      xmit.
1679  *
1680  *      Management xmit state machine consists of two states:
1681  *              IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1682  *              IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1683  **/
1684 static int
1685 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1686 {
1687
1688         debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1689                 conn->id, mtask->xmstate, mtask->itt);
1690
1691         if (mtask->xmstate & XMSTATE_IMM_HDR) {
1692                 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1693                 if (mtask->data_count)
1694                         mtask->xmstate |= XMSTATE_IMM_DATA;
1695                 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1696                         mtask->xmstate |= XMSTATE_IMM_HDR;
1697                         if (mtask->data_count)
1698                                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1699                         return -EAGAIN;
1700                 }
1701         }
1702
1703         if (mtask->xmstate & XMSTATE_IMM_DATA) {
1704                 BUG_ON(!mtask->data_count);
1705                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1706                 /* FIXME: implement.
1707                  * Virtual buffer could be spreaded across multiple pages...
1708                  */
1709                 do {
1710                         if (iscsi_sendpage(conn, &mtask->sendbuf,
1711                                    &mtask->data_count, &mtask->sent)) {
1712                                 mtask->xmstate |= XMSTATE_IMM_DATA;
1713                                 return -EAGAIN;
1714                         }
1715                 } while (mtask->data_count);
1716         }
1717
1718         BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1719         return 0;
1720 }
1721
1722 static inline int
1723 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1724 {
1725         ctask->xmstate &= ~XMSTATE_R_HDR;
1726         if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1727                 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1728                 return 0; /* wait for Data-In */
1729         }
1730         ctask->xmstate |= XMSTATE_R_HDR;
1731         return -EAGAIN;
1732 }
1733
1734 static inline int
1735 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1736 {
1737         ctask->xmstate &= ~XMSTATE_W_HDR;
1738         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1739                 ctask->xmstate |= XMSTATE_W_HDR;
1740                 return -EAGAIN;
1741         }
1742         return 0;
1743 }
1744
1745 static inline int
1746 handle_xmstate_data_digest(struct iscsi_conn *conn,
1747                            struct iscsi_cmd_task *ctask)
1748 {
1749         ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1750         debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1751         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1752                                     &ctask->datadigest, 0)) {
1753                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1754                 debug_tcp("resent data digest 0x%x fail!\n",
1755                           ctask->datadigest);
1756                 return -EAGAIN;
1757         }
1758         return 0;
1759 }
1760
1761 static inline int
1762 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1763 {
1764         BUG_ON(!ctask->imm_count);
1765         ctask->xmstate &= ~XMSTATE_IMM_DATA;
1766
1767         if (conn->datadgst_en) {
1768                 iscsi_data_digest_init(conn, ctask);
1769                 ctask->immdigest = 0;
1770         }
1771
1772         for (;;) {
1773                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1774                                    &ctask->sent)) {
1775                         ctask->xmstate |= XMSTATE_IMM_DATA;
1776                         if (conn->datadgst_en) {
1777                                 crypto_digest_final(conn->data_tx_tfm,
1778                                                 (u8*)&ctask->immdigest);
1779                                 debug_tcp("tx imm sendpage fail 0x%x\n",
1780                                           ctask->datadigest);
1781                         }
1782                         return -EAGAIN;
1783                 }
1784                 if (conn->datadgst_en)
1785                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1786
1787                 if (!ctask->imm_count)
1788                         break;
1789                 iscsi_buf_init_sg(&ctask->sendbuf,
1790                                   &ctask->sg[ctask->sg_count++]);
1791         }
1792
1793         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1794                 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1795                                             &ctask->immdigest, 1)) {
1796                         debug_tcp("sending imm digest 0x%x fail!\n",
1797                                   ctask->immdigest);
1798                         return -EAGAIN;
1799                 }
1800                 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1801         }
1802
1803         return 0;
1804 }
1805
1806 static inline int
1807 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1808 {
1809         struct iscsi_data_task *dtask;
1810
1811         ctask->xmstate |= XMSTATE_UNS_DATA;
1812         if (ctask->xmstate & XMSTATE_UNS_INIT) {
1813                 iscsi_unsolicit_data_init(conn, ctask);
1814                 BUG_ON(!ctask->dtask);
1815                 dtask = ctask->dtask;
1816
1817                 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1818         }
1819         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1820                 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1821                 ctask->xmstate |= XMSTATE_UNS_HDR;
1822                 return -EAGAIN;
1823         }
1824
1825         debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1826                    ctask->itt, ctask->unsol_count, ctask->sent);
1827         return 0;
1828 }
1829
1830 static inline int
1831 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1832 {
1833         struct iscsi_data_task *dtask = ctask->dtask;
1834
1835         BUG_ON(!ctask->data_count);
1836         ctask->xmstate &= ~XMSTATE_UNS_DATA;
1837
1838         if (conn->datadgst_en) {
1839                 iscsi_data_digest_init(conn, ctask);
1840                 dtask->digest = 0;
1841         }
1842
1843         for (;;) {
1844                 int start = ctask->sent;
1845
1846                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1847                                    &ctask->sent)) {
1848                         ctask->unsol_count -= ctask->sent - start;
1849                         ctask->xmstate |= XMSTATE_UNS_DATA;
1850                         /* will continue with this ctask later.. */
1851                         if (conn->datadgst_en) {
1852                                 crypto_digest_final(conn->data_tx_tfm,
1853                                                 (u8 *)&dtask->digest);
1854                                 debug_tcp("tx uns data fail 0x%x\n",
1855                                           dtask->digest);
1856                         }
1857                         return -EAGAIN;
1858                 }
1859
1860                 BUG_ON(ctask->sent > ctask->total_length);
1861                 ctask->unsol_count -= ctask->sent - start;
1862
1863                 /*
1864                  * XXX:we may run here with un-initial sendbuf.
1865                  * so pass it
1866                  */
1867                 if (conn->datadgst_en && ctask->sent - start > 0)
1868                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1869
1870                 if (!ctask->data_count)
1871                         break;
1872                 iscsi_buf_init_sg(&ctask->sendbuf,
1873                                   &ctask->sg[ctask->sg_count++]);
1874         }
1875         BUG_ON(ctask->unsol_count < 0);
1876
1877         /*
1878          * Done with the Data-Out. Next, check if we need
1879          * to send another unsolicited Data-Out.
1880          */
1881         if (ctask->unsol_count) {
1882                 if (conn->datadgst_en) {
1883                         if (iscsi_digest_final_send(conn, ctask,
1884                                                     &dtask->digestbuf,
1885                                                     &dtask->digest, 1)) {
1886                                 debug_tcp("send uns digest 0x%x fail\n",
1887                                           dtask->digest);
1888                                 return -EAGAIN;
1889                         }
1890                         debug_tcp("sending uns digest 0x%x, more uns\n",
1891                                   dtask->digest);
1892                 }
1893                 ctask->xmstate |= XMSTATE_UNS_INIT;
1894                 return 1;
1895         }
1896
1897         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1898                 if (iscsi_digest_final_send(conn, ctask,
1899                                             &dtask->digestbuf,
1900                                             &dtask->digest, 1)) {
1901                         debug_tcp("send last uns digest 0x%x fail\n",
1902                                    dtask->digest);
1903                         return -EAGAIN;
1904                 }
1905                 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1906         }
1907
1908         return 0;
1909 }
1910
1911 static inline int
1912 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1913 {
1914         struct iscsi_session *session = conn->session;
1915         struct iscsi_r2t_info *r2t = ctask->r2t;
1916         struct iscsi_data_task *dtask = r2t->dtask;
1917         int left;
1918
1919         ctask->xmstate &= ~XMSTATE_SOL_DATA;
1920         ctask->dtask = dtask;
1921
1922         if (conn->datadgst_en) {
1923                 iscsi_data_digest_init(conn, ctask);
1924                 dtask->digest = 0;
1925         }
1926 solicit_again:
1927         /*
1928          * send Data-Out whitnin this R2T sequence.
1929          */
1930         if (!r2t->data_count)
1931                 goto data_out_done;
1932
1933         if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1934                 ctask->xmstate |= XMSTATE_SOL_DATA;
1935                 /* will continue with this ctask later.. */
1936                 if (conn->datadgst_en) {
1937                         crypto_digest_final(conn->data_tx_tfm,
1938                                           (u8 *)&dtask->digest);
1939                         debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1940                 }
1941                 return -EAGAIN;
1942         }
1943
1944         BUG_ON(r2t->data_count < 0);
1945         if (conn->datadgst_en)
1946                 iscsi_buf_data_digest_update(conn, &r2t->sendbuf);
1947
1948         if (r2t->data_count) {
1949                 BUG_ON(ctask->sc->use_sg == 0);
1950                 if (!iscsi_buf_left(&r2t->sendbuf)) {
1951                         BUG_ON(ctask->bad_sg == r2t->sg);
1952                         iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1953                         r2t->sg += 1;
1954                 }
1955                 goto solicit_again;
1956         }
1957
1958 data_out_done:
1959         /*
1960          * Done with this Data-Out. Next, check if we have
1961          * to send another Data-Out for this R2T.
1962          */
1963         BUG_ON(r2t->data_length - r2t->sent < 0);
1964         left = r2t->data_length - r2t->sent;
1965         if (left) {
1966                 if (conn->datadgst_en) {
1967                         if (iscsi_digest_final_send(conn, ctask,
1968                                                     &dtask->digestbuf,
1969                                                     &dtask->digest, 1)) {
1970                                 debug_tcp("send r2t data digest 0x%x"
1971                                           "fail\n", dtask->digest);
1972                                 return -EAGAIN;
1973                         }
1974                         debug_tcp("r2t data send digest 0x%x\n",
1975                                   dtask->digest);
1976                 }
1977                 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1978                 ctask->xmstate |= XMSTATE_SOL_DATA;
1979                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
1980                 return 1;
1981         }
1982
1983         /*
1984          * Done with this R2T. Check if there are more
1985          * outstanding R2Ts ready to be processed.
1986          */
1987         BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
1988         if (conn->datadgst_en) {
1989                 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1990                                             &dtask->digest, 1)) {
1991                         debug_tcp("send last r2t data digest 0x%x"
1992                                   "fail\n", dtask->digest);
1993                         return -EAGAIN;
1994                 }
1995                 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1996         }
1997
1998         ctask->r2t_data_count -= r2t->data_length;
1999         ctask->r2t = NULL;
2000         spin_lock_bh(&session->lock);
2001         __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2002         spin_unlock_bh(&session->lock);
2003         if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2004                 ctask->r2t = r2t;
2005                 ctask->xmstate |= XMSTATE_SOL_DATA;
2006                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2007                 return 1;
2008         }
2009
2010         return 0;
2011 }
2012
2013 static inline int
2014 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2015 {
2016         struct iscsi_data_task *dtask = ctask->dtask;
2017         int sent;
2018
2019         ctask->xmstate &= ~XMSTATE_W_PAD;
2020         iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2021                             ctask->pad_count);
2022         if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2023                 ctask->xmstate |= XMSTATE_W_PAD;
2024                 return -EAGAIN;
2025         }
2026
2027         if (conn->datadgst_en) {
2028                 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
2029                 /* imm data? */
2030                 if (!dtask) {
2031                         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2032                                                     &ctask->immdigest, 1)) {
2033                                 debug_tcp("send padding digest 0x%x"
2034                                           "fail!\n", ctask->immdigest);
2035                                 return -EAGAIN;
2036                         }
2037                         debug_tcp("done with padding, digest 0x%x\n",
2038                                   ctask->datadigest);
2039                 } else {
2040                         if (iscsi_digest_final_send(conn, ctask,
2041                                                     &dtask->digestbuf,
2042                                                     &dtask->digest, 1)) {
2043                                 debug_tcp("send padding digest 0x%x"
2044                                           "fail\n", dtask->digest);
2045                                 return -EAGAIN;
2046                         }
2047                         debug_tcp("done with padding, digest 0x%x\n",
2048                                   dtask->digest);
2049                 }
2050         }
2051
2052         return 0;
2053 }
2054
2055 static int
2056 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2057 {
2058         int rc = 0;
2059
2060         debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2061                 conn->id, ctask->xmstate, ctask->itt);
2062
2063         /*
2064          * serialize with TMF AbortTask
2065          */
2066         if (ctask->mtask)
2067                 return rc;
2068
2069         if (ctask->xmstate & XMSTATE_R_HDR) {
2070                 rc = handle_xmstate_r_hdr(conn, ctask);
2071                 return rc;
2072         }
2073
2074         if (ctask->xmstate & XMSTATE_W_HDR) {
2075                 rc = handle_xmstate_w_hdr(conn, ctask);
2076                 if (rc)
2077                         return rc;
2078         }
2079
2080         /* XXX: for data digest xmit recover */
2081         if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2082                 rc = handle_xmstate_data_digest(conn, ctask);
2083                 if (rc)
2084                         return rc;
2085         }
2086
2087         if (ctask->xmstate & XMSTATE_IMM_DATA) {
2088                 rc = handle_xmstate_imm_data(conn, ctask);
2089                 if (rc)
2090                         return rc;
2091         }
2092
2093         if (ctask->xmstate & XMSTATE_UNS_HDR) {
2094                 BUG_ON(!ctask->unsol_count);
2095                 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2096 unsolicit_head_again:
2097                 rc = handle_xmstate_uns_hdr(conn, ctask);
2098                 if (rc)
2099                         return rc;
2100         }
2101
2102         if (ctask->xmstate & XMSTATE_UNS_DATA) {
2103                 rc = handle_xmstate_uns_data(conn, ctask);
2104                 if (rc == 1)
2105                         goto unsolicit_head_again;
2106                 else if (rc)
2107                         return rc;
2108                 goto done;
2109         }
2110
2111         if (ctask->xmstate & XMSTATE_SOL_HDR) {
2112                 struct iscsi_r2t_info *r2t;
2113
2114                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2115                 ctask->xmstate |= XMSTATE_SOL_DATA;
2116                 if (!ctask->r2t)
2117                         __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2118                                     sizeof(void*));
2119 solicit_head_again:
2120                 r2t = ctask->r2t;
2121
2122                 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2123                         ctask->xmstate &= ~XMSTATE_SOL_DATA;
2124                         ctask->xmstate |= XMSTATE_SOL_HDR;
2125                         return -EAGAIN;
2126                 }
2127
2128                 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2129                         r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2130                         r2t->sent);
2131         }
2132
2133         if (ctask->xmstate & XMSTATE_SOL_DATA) {
2134                 rc = handle_xmstate_sol_data(conn, ctask);
2135                 if (rc == 1)
2136                         goto solicit_head_again;
2137                 if (rc)
2138                         return rc;
2139         }
2140
2141 done:
2142         /*
2143          * Last thing to check is whether we need to send write
2144          * padding. Note that we check for xmstate equality, not just the bit.
2145          */
2146         if (ctask->xmstate == XMSTATE_W_PAD)
2147                 rc = handle_xmstate_w_pad(conn, ctask);
2148
2149         return rc;
2150 }
2151
2152 /**
2153  * iscsi_data_xmit - xmit any command into the scheduled connection
2154  * @conn: iscsi connection
2155  *
2156  * Notes:
2157  *      The function can return -EAGAIN in which case the caller must
2158  *      re-schedule it again later or recover. '0' return code means
2159  *      successful xmit.
2160  **/
2161 static int
2162 iscsi_data_xmit(struct iscsi_conn *conn)
2163 {
2164         if (unlikely(conn->suspend_tx)) {
2165                 debug_tcp("conn %d Tx suspended!\n", conn->id);
2166                 return 0;
2167         }
2168
2169         /*
2170          * Transmit in the following order:
2171          *
2172          * 1) un-finished xmit (ctask or mtask)
2173          * 2) immediate control PDUs
2174          * 3) write data
2175          * 4) SCSI commands
2176          * 5) non-immediate control PDUs
2177          *
2178          * No need to lock around __kfifo_get as long as
2179          * there's one producer and one consumer.
2180          */
2181
2182         BUG_ON(conn->ctask && conn->mtask);
2183
2184         if (conn->ctask) {
2185                 if (iscsi_ctask_xmit(conn, conn->ctask))
2186                         goto again;
2187                 /* done with this in-progress ctask */
2188                 conn->ctask = NULL;
2189         }
2190         if (conn->mtask) {
2191                 if (iscsi_mtask_xmit(conn, conn->mtask))
2192                         goto again;
2193                 /* done with this in-progress mtask */
2194                 conn->mtask = NULL;
2195         }
2196
2197         /* process immediate first */
2198         if (unlikely(__kfifo_len(conn->immqueue))) {
2199                 struct iscsi_session *session = conn->session;
2200                 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2201                                    sizeof(void*))) {
2202                         if (iscsi_mtask_xmit(conn, conn->mtask))
2203                                 goto again;
2204
2205                         if (conn->mtask->hdr.itt ==
2206                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2207                                 spin_lock_bh(&session->lock);
2208                                 __kfifo_put(session->mgmtpool.queue,
2209                                             (void*)&conn->mtask, sizeof(void*));
2210                                 spin_unlock_bh(&session->lock);
2211                         }
2212                 }
2213                 /* done with this mtask */
2214                 conn->mtask = NULL;
2215         }
2216
2217         /* process write queue */
2218         while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2219                            sizeof(void*))) {
2220                 if (iscsi_ctask_xmit(conn, conn->ctask))
2221                         goto again;
2222         }
2223
2224         /* process command queue */
2225         while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2226                            sizeof(void*))) {
2227                 if (iscsi_ctask_xmit(conn, conn->ctask))
2228                         goto again;
2229         }
2230         /* done with this ctask */
2231         conn->ctask = NULL;
2232
2233         /* process the rest control plane PDUs, if any */
2234         if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2235                 struct iscsi_session *session = conn->session;
2236
2237                 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2238                                    sizeof(void*))) {
2239                         if (iscsi_mtask_xmit(conn, conn->mtask))
2240                                 goto again;
2241
2242                         if (conn->mtask->hdr.itt ==
2243                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2244                                 spin_lock_bh(&session->lock);
2245                                 __kfifo_put(session->mgmtpool.queue,
2246                                             (void*)&conn->mtask,
2247                                             sizeof(void*));
2248                                 spin_unlock_bh(&session->lock);
2249                         }
2250                 }
2251                 /* done with this mtask */
2252                 conn->mtask = NULL;
2253         }
2254
2255         return 0;
2256
2257 again:
2258         if (unlikely(conn->suspend_tx))
2259                 return 0;
2260
2261         return -EAGAIN;
2262 }
2263
2264 static void
2265 iscsi_xmitworker(void *data)
2266 {
2267         struct iscsi_conn *conn = data;
2268
2269         /*
2270          * serialize Xmit worker on a per-connection basis.
2271          */
2272         down(&conn->xmitsema);
2273         if (iscsi_data_xmit(conn))
2274                 schedule_work(&conn->xmitwork);
2275         up(&conn->xmitsema);
2276 }
2277
2278 #define FAILURE_BAD_HOST                1
2279 #define FAILURE_SESSION_FAILED          2
2280 #define FAILURE_SESSION_FREED           3
2281 #define FAILURE_WINDOW_CLOSED           4
2282 #define FAILURE_SESSION_TERMINATE       5
2283
2284 static int
2285 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2286 {
2287         struct Scsi_Host *host;
2288         int reason = 0;
2289         struct iscsi_session *session;
2290         struct iscsi_conn *conn = NULL;
2291         struct iscsi_cmd_task *ctask = NULL;
2292
2293         sc->scsi_done = done;
2294         sc->result = 0;
2295
2296         host = sc->device->host;
2297         session = iscsi_hostdata(host->hostdata);
2298         BUG_ON(host != session->host);
2299
2300         spin_lock(&session->lock);
2301
2302         if (session->state != ISCSI_STATE_LOGGED_IN) {
2303                 if (session->state == ISCSI_STATE_FAILED) {
2304                         reason = FAILURE_SESSION_FAILED;
2305                         goto reject;
2306                 } else if (session->state == ISCSI_STATE_TERMINATE) {
2307                         reason = FAILURE_SESSION_TERMINATE;
2308                         goto fault;
2309                 }
2310                 reason = FAILURE_SESSION_FREED;
2311                 goto fault;
2312         }
2313
2314         /*
2315          * Check for iSCSI window and take care of CmdSN wrap-around
2316          */
2317         if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2318                 reason = FAILURE_WINDOW_CLOSED;
2319                 goto reject;
2320         }
2321
2322         conn = session->leadconn;
2323
2324         __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2325         BUG_ON(ctask->sc);
2326
2327         sc->SCp.phase = session->age;
2328         sc->SCp.ptr = (char*)ctask;
2329         iscsi_cmd_init(conn, ctask, sc);
2330
2331         __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2332         debug_scsi(
2333                "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2334                 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2335                 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2336                 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2337         spin_unlock(&session->lock);
2338
2339         if (!in_interrupt() && !down_trylock(&conn->xmitsema)) {
2340                 spin_unlock_irq(host->host_lock);
2341                 if (iscsi_data_xmit(conn))
2342                         schedule_work(&conn->xmitwork);
2343                 up(&conn->xmitsema);
2344                 spin_lock_irq(host->host_lock);
2345         } else
2346                 schedule_work(&conn->xmitwork);
2347
2348         return 0;
2349
2350 reject:
2351         spin_unlock(&session->lock);
2352         debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2353         return SCSI_MLQUEUE_HOST_BUSY;
2354
2355 fault:
2356         spin_unlock(&session->lock);
2357         printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2358                sc->cmnd[0], reason);
2359         sc->sense_buffer[0] = 0x70;
2360         sc->sense_buffer[2] = NOT_READY;
2361         sc->sense_buffer[7] = 0x6;
2362         sc->sense_buffer[12] = 0x08;
2363         sc->sense_buffer[13] = 0x00;
2364         sc->result = (DID_NO_CONNECT << 16);
2365         sc->resid = sc->request_bufflen;
2366         sc->scsi_done(sc);
2367         return 0;
2368 }
2369
2370 static int
2371 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2372 {
2373         int i;
2374
2375         *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2376         if (*items == NULL)
2377                 return -ENOMEM;
2378
2379         q->max = max;
2380         q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2381         if (q->pool == NULL) {
2382                 kfree(*items);
2383                 return -ENOMEM;
2384         }
2385
2386         q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2387                               GFP_KERNEL, NULL);
2388         if (q->queue == ERR_PTR(-ENOMEM)) {
2389                 kfree(q->pool);
2390                 kfree(*items);
2391                 return -ENOMEM;
2392         }
2393
2394         for (i = 0; i < max; i++) {
2395                 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2396                 if (q->pool[i] == NULL) {
2397                         int j;
2398
2399                         for (j = 0; j < i; j++)
2400                                 kfree(q->pool[j]);
2401
2402                         kfifo_free(q->queue);
2403                         kfree(q->pool);
2404                         kfree(*items);
2405                         return -ENOMEM;
2406                 }
2407                 memset(q->pool[i], 0, item_size);
2408                 (*items)[i] = q->pool[i];
2409                 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2410         }
2411         return 0;
2412 }
2413
2414 static void
2415 iscsi_pool_free(struct iscsi_queue *q, void **items)
2416 {
2417         int i;
2418
2419         for (i = 0; i < q->max; i++)
2420                 kfree(items[i]);
2421         kfree(q->pool);
2422         kfree(items);
2423 }
2424
2425 static iscsi_connh_t
2426 iscsi_conn_create(iscsi_sessionh_t sessionh, uint32_t conn_idx)
2427 {
2428         struct iscsi_session *session = iscsi_ptr(sessionh);
2429         struct iscsi_conn *conn = NULL;
2430
2431         conn = kmalloc(sizeof(struct iscsi_conn), GFP_KERNEL);
2432         if (conn == NULL)
2433                 goto conn_alloc_fail;
2434         memset(conn, 0, sizeof(struct iscsi_conn));
2435
2436         conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2437         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2438         conn->id = conn_idx;
2439         conn->exp_statsn = 0;
2440         conn->tmabort_state = TMABORT_INITIAL;
2441
2442         /* initial operational parameters */
2443         conn->hdr_size = sizeof(struct iscsi_hdr);
2444         conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2445         conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2446
2447         spin_lock_init(&conn->lock);
2448
2449         /* initialize general xmit PDU commands queue */
2450         conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2451                                         GFP_KERNEL, NULL);
2452         if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2453                 goto xmitqueue_alloc_fail;
2454
2455         /* initialize write response PDU commands queue */
2456         conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2457                                         GFP_KERNEL, NULL);
2458         if (conn->writequeue == ERR_PTR(-ENOMEM))
2459                 goto writequeue_alloc_fail;
2460
2461         /* initialize general immediate & non-immediate PDU commands queue */
2462         conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2463                                         GFP_KERNEL, NULL);
2464         if (conn->immqueue == ERR_PTR(-ENOMEM))
2465                 goto immqueue_alloc_fail;
2466
2467         conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2468                                         GFP_KERNEL, NULL);
2469         if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2470                 goto mgmtqueue_alloc_fail;
2471
2472         INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2473
2474         /* allocate login_mtask used for the login/text sequences */
2475         spin_lock_bh(&session->lock);
2476         if (!__kfifo_get(session->mgmtpool.queue,
2477                          (void*)&conn->login_mtask,
2478                          sizeof(void*))) {
2479                 spin_unlock_bh(&session->lock);
2480                 goto login_mtask_alloc_fail;
2481         }
2482         spin_unlock_bh(&session->lock);
2483
2484         /* allocate initial PDU receive place holder */
2485         if (conn->data_size <= PAGE_SIZE)
2486                 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2487         else
2488                 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2489                                         get_order(conn->data_size));
2490         if (!conn->data)
2491                 goto max_recv_dlenght_alloc_fail;
2492
2493         init_timer(&conn->tmabort_timer);
2494         init_MUTEX(&conn->xmitsema);
2495         init_waitqueue_head(&conn->ehwait);
2496
2497         return iscsi_handle(conn);
2498
2499 max_recv_dlenght_alloc_fail:
2500         spin_lock_bh(&session->lock);
2501         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2502                     sizeof(void*));
2503         spin_unlock_bh(&session->lock);
2504 login_mtask_alloc_fail:
2505         kfifo_free(conn->mgmtqueue);
2506 mgmtqueue_alloc_fail:
2507         kfifo_free(conn->immqueue);
2508 immqueue_alloc_fail:
2509         kfifo_free(conn->writequeue);
2510 writequeue_alloc_fail:
2511         kfifo_free(conn->xmitqueue);
2512 xmitqueue_alloc_fail:
2513         kfree(conn);
2514 conn_alloc_fail:
2515         return iscsi_handle(NULL);
2516 }
2517
2518 static void
2519 iscsi_conn_destroy(iscsi_connh_t connh)
2520 {
2521         struct iscsi_conn *conn = iscsi_ptr(connh);
2522         struct iscsi_session *session = conn->session;
2523
2524         down(&conn->xmitsema);
2525         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2526         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2527                 struct sock *sk = conn->sock->sk;
2528
2529                 /*
2530                  * conn_start() has never been called!
2531                  * need to cleanup the socket.
2532                  */
2533                 write_lock_bh(&sk->sk_callback_lock);
2534                 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2535                 write_unlock_bh(&sk->sk_callback_lock);
2536
2537                 sock_hold(conn->sock->sk);
2538                 iscsi_conn_restore_callbacks(conn);
2539                 sock_put(conn->sock->sk);
2540                 sock_release(conn->sock);
2541                 conn->sock = NULL;
2542         }
2543
2544         spin_lock_bh(&session->lock);
2545         conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2546         if (session->leadconn == conn) {
2547                 /*
2548                  * leading connection? then give up on recovery.
2549                  */
2550                 session->state = ISCSI_STATE_TERMINATE;
2551                 wake_up(&conn->ehwait);
2552         }
2553         spin_unlock_bh(&session->lock);
2554
2555         up(&conn->xmitsema);
2556
2557         /*
2558          * Block until all in-progress commands for this connection
2559          * time out or fail.
2560          */
2561         for (;;) {
2562                 spin_lock_bh(&conn->lock);
2563                 if (!session->host->host_busy) { /* OK for ERL == 0 */
2564                         spin_unlock_bh(&conn->lock);
2565                         break;
2566                 }
2567                 spin_unlock_bh(&conn->lock);
2568                 msleep_interruptible(500);
2569                 printk("conn_destroy(): host_busy %d host_failed %d\n",
2570                         session->host->host_busy, session->host->host_failed);
2571                 /*
2572                  * force eh_abort() to unblock
2573                  */
2574                 wake_up(&conn->ehwait);
2575         }
2576
2577         /* now free crypto */
2578         if (conn->hdrdgst_en || conn->datadgst_en) {
2579                 if (conn->tx_tfm)
2580                         crypto_free_tfm(conn->tx_tfm);
2581                 if (conn->rx_tfm)
2582                         crypto_free_tfm(conn->rx_tfm);
2583                 if (conn->data_tx_tfm)
2584                         crypto_free_tfm(conn->data_tx_tfm);
2585                 if (conn->data_rx_tfm)
2586                         crypto_free_tfm(conn->data_rx_tfm);
2587         }
2588
2589         /* free conn->data, size = MaxRecvDataSegmentLength */
2590         if (conn->data_size <= PAGE_SIZE)
2591                 kfree(conn->data);
2592         else
2593                 free_pages((unsigned long)conn->data,
2594                                         get_order(conn->data_size));
2595
2596         spin_lock_bh(&session->lock);
2597         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2598                     sizeof(void*));
2599         list_del(&conn->item);
2600         if (list_empty(&session->connections))
2601                 session->leadconn = NULL;
2602         if (session->leadconn && session->leadconn == conn)
2603                 session->leadconn = container_of(session->connections.next,
2604                         struct iscsi_conn, item);
2605
2606         if (session->leadconn == NULL)
2607                 /* none connections exits.. reset sequencing */
2608                 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2609         spin_unlock_bh(&session->lock);
2610
2611         kfifo_free(conn->xmitqueue);
2612         kfifo_free(conn->writequeue);
2613         kfifo_free(conn->immqueue);
2614         kfifo_free(conn->mgmtqueue);
2615         kfree(conn);
2616 }
2617
2618 static int
2619 iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2620                 uint32_t transport_fd, int is_leading)
2621 {
2622         struct iscsi_session *session = iscsi_ptr(sessionh);
2623         struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2624         struct sock *sk;
2625         struct socket *sock;
2626         int err;
2627
2628         /* lookup for existing socket */
2629         sock = sockfd_lookup(transport_fd, &err);
2630         if (!sock) {
2631                 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2632                 return -EEXIST;
2633         }
2634
2635         /* lookup for existing connection */
2636         spin_lock_bh(&session->lock);
2637         list_for_each_entry(tmp, &session->connections, item) {
2638                 if (tmp == conn) {
2639                         if (conn->c_stage != ISCSI_CONN_STOPPED ||
2640                             conn->stop_stage == STOP_CONN_TERM) {
2641                                 printk(KERN_ERR "iscsi_tcp: can't bind "
2642                                        "non-stopped connection (%d:%d)\n",
2643                                        conn->c_stage, conn->stop_stage);
2644                                 spin_unlock_bh(&session->lock);
2645                                 return -EIO;
2646                         }
2647                         break;
2648                 }
2649         }
2650         if (tmp != conn) {
2651                 /* bind new iSCSI connection to session */
2652                 conn->session = session;
2653
2654                 list_add(&conn->item, &session->connections);
2655         }
2656         spin_unlock_bh(&session->lock);
2657
2658         if (conn->stop_stage != STOP_CONN_SUSPEND) {
2659                 /* bind iSCSI connection and socket */
2660                 conn->sock = sock;
2661
2662                 /* setup Socket parameters */
2663                 sk = sock->sk;
2664                 sk->sk_reuse = 1;
2665                 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2666                 sk->sk_allocation = GFP_ATOMIC;
2667
2668                 /* FIXME: disable Nagle's algorithm */
2669
2670                 /*
2671                  * Intercept TCP callbacks for sendfile like receive
2672                  * processing.
2673                  */
2674                 iscsi_conn_set_callbacks(conn);
2675
2676                 /*
2677                  * set receive state machine into initial state
2678                  */
2679                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2680         }
2681
2682         if (is_leading)
2683                 session->leadconn = conn;
2684
2685         /*
2686          * Unblock xmitworker(), Login Phase will pass through.
2687          */
2688         clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2689         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2690
2691         return 0;
2692 }
2693
2694 static int
2695 iscsi_conn_start(iscsi_connh_t connh)
2696 {
2697         struct iscsi_conn *conn = iscsi_ptr(connh);
2698         struct iscsi_session *session = conn->session;
2699         struct sock *sk;
2700
2701         /* FF phase warming up... */
2702
2703         if (session == NULL) {
2704                 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2705                 return -EPERM;
2706         }
2707
2708         sk = conn->sock->sk;
2709
2710         write_lock_bh(&sk->sk_callback_lock);
2711         spin_lock_bh(&session->lock);
2712         conn->c_stage = ISCSI_CONN_STARTED;
2713         session->state = ISCSI_STATE_LOGGED_IN;
2714
2715         switch(conn->stop_stage) {
2716         case STOP_CONN_RECOVER:
2717                 /*
2718                  * unblock eh_abort() if it is blocked. re-try all
2719                  * commands after successful recovery
2720                  */
2721                 session->conn_cnt++;
2722                 conn->stop_stage = 0;
2723                 conn->tmabort_state = TMABORT_INITIAL;
2724                 session->age++;
2725                 wake_up(&conn->ehwait);
2726                 break;
2727         case STOP_CONN_TERM:
2728                 session->conn_cnt++;
2729                 conn->stop_stage = 0;
2730                 break;
2731         case STOP_CONN_SUSPEND:
2732                 conn->stop_stage = 0;
2733                 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2734                 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2735                 break;
2736         default:
2737                 break;
2738         }
2739         spin_unlock_bh(&session->lock);
2740         write_unlock_bh(&sk->sk_callback_lock);
2741
2742         return 0;
2743 }
2744
2745 static void
2746 iscsi_conn_stop(iscsi_connh_t connh, int flag)
2747 {
2748         struct iscsi_conn *conn = iscsi_ptr(connh);
2749         struct iscsi_session *session = conn->session;
2750         struct sock *sk;
2751         unsigned long flags;
2752
2753         BUG_ON(!conn->sock);
2754         sk = conn->sock->sk;
2755         write_lock_bh(&sk->sk_callback_lock);
2756         set_bit(SUSPEND_BIT, &conn->suspend_rx);
2757         write_unlock_bh(&sk->sk_callback_lock);
2758
2759         down(&conn->xmitsema);
2760
2761         spin_lock_irqsave(session->host->host_lock, flags);
2762         spin_lock(&session->lock);
2763         conn->stop_stage = flag;
2764         conn->c_stage = ISCSI_CONN_STOPPED;
2765         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2766
2767         if (flag != STOP_CONN_SUSPEND)
2768                 session->conn_cnt--;
2769
2770         if (session->conn_cnt == 0 || session->leadconn == conn)
2771                 session->state = ISCSI_STATE_FAILED;
2772
2773         spin_unlock(&session->lock);
2774         spin_unlock_irqrestore(session->host->host_lock, flags);
2775
2776         if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2777                 struct iscsi_cmd_task *ctask;
2778                 struct iscsi_mgmt_task *mtask;
2779
2780                 /*
2781                  * Socket must go now.
2782                  */
2783                 sock_hold(conn->sock->sk);
2784                 iscsi_conn_restore_callbacks(conn);
2785                 sock_put(conn->sock->sk);
2786
2787                 /*
2788                  * flush xmit queues.
2789                  */
2790                 spin_lock_bh(&session->lock);
2791                 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2792                             sizeof(void*)) ||
2793                         __kfifo_get(conn->xmitqueue, (void*)&ctask,
2794                             sizeof(void*))) {
2795                         struct iscsi_r2t_info *r2t;
2796
2797                         /*
2798                          * flush ctask's r2t queues
2799                          */
2800                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2801                                 sizeof(void*)))
2802                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2803                                             sizeof(void*));
2804
2805                         spin_unlock_bh(&session->lock);
2806                         local_bh_disable();
2807                         iscsi_ctask_cleanup(conn, ctask);
2808                         local_bh_enable();
2809                         spin_lock_bh(&session->lock);
2810                 }
2811                 conn->ctask = NULL;
2812                 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2813                            sizeof(void*)) ||
2814                         __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2815                            sizeof(void*))) {
2816                         __kfifo_put(session->mgmtpool.queue,
2817                                     (void*)&mtask, sizeof(void*));
2818                 }
2819                 conn->mtask = NULL;
2820                 spin_unlock_bh(&session->lock);
2821
2822                 /*
2823                  * release socket only after we stopped data_xmit()
2824                  * activity and flushed all outstandings
2825                  */
2826                 sock_release(conn->sock);
2827                 conn->sock = NULL;
2828
2829                 /*
2830                  * for connection level recovery we should not calculate
2831                  * header digest. conn->hdr_size used for optimization
2832                  * in hdr_extract() and will be re-negotiated at
2833                  * set_param() time.
2834                  */
2835                 if (flag == STOP_CONN_RECOVER)
2836                         conn->hdr_size = sizeof(struct iscsi_hdr);
2837         }
2838         up(&conn->xmitsema);
2839 }
2840
2841 static int
2842 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2843                         char *data, uint32_t data_size)
2844 {
2845         struct iscsi_session *session = conn->session;
2846         struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2847         struct iscsi_mgmt_task *mtask;
2848
2849         spin_lock_bh(&session->lock);
2850         if (session->state == ISCSI_STATE_TERMINATE) {
2851                 spin_unlock_bh(&session->lock);
2852                 return -EPERM;
2853         }
2854         if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2855             hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2856                 /*
2857                  * Login and Text are sent serially, in
2858                  * request-followed-by-response sequence.
2859                  * Same mtask can be used. Same ITT must be used.
2860                  * Note that login_mtask is preallocated at conn_create().
2861                  */
2862                 mtask = conn->login_mtask;
2863         else {
2864                 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2865                 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2866
2867                 if (!__kfifo_get(session->mgmtpool.queue,
2868                                  (void*)&mtask, sizeof(void*))) {
2869                         spin_unlock_bh(&session->lock);
2870                         return -ENOSPC;
2871                 }
2872         }
2873
2874         /*
2875          * pre-format CmdSN and ExpStatSN for outgoing PDU.
2876          */
2877         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2878                 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2879                            (session->age << AGE_SHIFT);
2880                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2881                 if (conn->c_stage == ISCSI_CONN_STARTED &&
2882                     !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2883                         session->cmdsn++;
2884         } else
2885                 /* do not advance CmdSN */
2886                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2887
2888         nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2889
2890         memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2891
2892         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE ||
2893             conn->stop_stage == STOP_CONN_RECOVER)
2894                 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2895                                     sizeof(struct iscsi_hdr));
2896         else
2897                 /* this will update header digest */
2898                 iscsi_buf_init_hdr(conn, &mtask->headbuf, (char*)&mtask->hdr,
2899                                     (u8 *)mtask->hdrext);
2900
2901         spin_unlock_bh(&session->lock);
2902
2903         if (data_size) {
2904                 memcpy(mtask->data, data, data_size);
2905                 mtask->data_count = data_size;
2906         } else
2907                 mtask->data_count = 0;
2908
2909         mtask->xmstate = XMSTATE_IMM_HDR;
2910
2911         if (mtask->data_count) {
2912                 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2913                                     mtask->data_count);
2914         }
2915
2916         debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2917                    hdr->opcode, hdr->itt, data_size);
2918
2919         /*
2920          * since send_pdu() could be called at least from two contexts,
2921          * we need to serialize __kfifo_put, so we don't have to take
2922          * additional lock on fast data-path
2923          */
2924         if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2925                 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2926         else
2927                 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2928
2929         schedule_work(&conn->xmitwork);
2930
2931         return 0;
2932 }
2933
2934 static int
2935 iscsi_eh_host_reset(struct scsi_cmnd *sc)
2936 {
2937         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2938         struct iscsi_conn *conn = ctask->conn;
2939         struct iscsi_session *session = conn->session;
2940
2941         spin_lock_bh(&session->lock);
2942         if (session->state == ISCSI_STATE_TERMINATE) {
2943                 debug_scsi("failing host reset: session terminated "
2944                            "[CID %d age %d]", conn->id, session->age);
2945                 spin_unlock_bh(&session->lock);
2946                 return FAILED;
2947         }
2948         spin_unlock_bh(&session->lock);
2949
2950         debug_scsi("failing connection CID %d due to SCSI host reset "
2951                    "[itt 0x%x age %d]", conn->id, ctask->itt,
2952                    session->age);
2953         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2954
2955         return SUCCESS;
2956 }
2957
2958 static void
2959 iscsi_tmabort_timedout(unsigned long data)
2960 {
2961         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2962         struct iscsi_conn *conn = ctask->conn;
2963         struct iscsi_session *session = conn->session;
2964
2965         spin_lock(&session->lock);
2966         if (conn->tmabort_state == TMABORT_INITIAL) {
2967                 __kfifo_put(session->mgmtpool.queue,
2968                                 (void*)&ctask->mtask, sizeof(void*));
2969                 conn->tmabort_state = TMABORT_TIMEDOUT;
2970                 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2971                         (long)ctask->sc, ctask->itt);
2972                 /* unblock eh_abort() */
2973                 wake_up(&conn->ehwait);
2974         }
2975         spin_unlock(&session->lock);
2976 }
2977
2978 static int
2979 iscsi_eh_abort(struct scsi_cmnd *sc)
2980 {
2981         int rc;
2982         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2983         struct iscsi_conn *conn = ctask->conn;
2984         struct iscsi_session *session = conn->session;
2985
2986         conn->eh_abort_cnt++;
2987         debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
2988
2989         /*
2990          * two cases for ERL=0 here:
2991          *
2992          * 1) connection-level failure;
2993          * 2) recovery due protocol error;
2994          */
2995         down(&conn->xmitsema);
2996         spin_lock_bh(&session->lock);
2997         if (session->state != ISCSI_STATE_LOGGED_IN) {
2998                 if (session->state == ISCSI_STATE_TERMINATE) {
2999                         spin_unlock_bh(&session->lock);
3000                         up(&conn->xmitsema);
3001                         goto failed;
3002                 }
3003                 spin_unlock_bh(&session->lock);
3004         } else {
3005                 struct iscsi_tm *hdr = &conn->tmhdr;
3006
3007                 /*
3008                  * Still LOGGED_IN...
3009                  */
3010
3011                 if (!ctask->sc || sc->SCp.phase != session->age) {
3012                         /*
3013                          * 1) ctask completed before time out. But session
3014                          *    is still ok => Happy Retry.
3015                          * 2) session was re-open during time out of ctask.
3016                          */
3017                         spin_unlock_bh(&session->lock);
3018                         up(&conn->xmitsema);
3019                         goto success;
3020                 }
3021                 conn->tmabort_state = TMABORT_INITIAL;
3022                 spin_unlock_bh(&session->lock);
3023
3024                 /*
3025                  * ctask timed out but session is OK
3026                  * ERL=0 requires task mgmt abort to be issued on each
3027                  * failed command. requests must be serialized.
3028                  */
3029                 memset(hdr, 0, sizeof(struct iscsi_tm));
3030                 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3031                 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3032                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3033                 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3034                 hdr->rtt = ctask->hdr.itt;
3035                 hdr->refcmdsn = ctask->hdr.cmdsn;
3036
3037                 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3038                                              NULL, 0);
3039                 if (rc) {
3040                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3041                         debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3042                 } else {
3043                         struct iscsi_r2t_info *r2t;
3044
3045                         /*
3046                          * TMF abort vs. TMF response race logic
3047                          */
3048                         spin_lock_bh(&session->lock);
3049                         ctask->mtask = (struct iscsi_mgmt_task *)
3050                                 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3051                                                         ISCSI_MGMT_ITT_OFFSET];
3052                         /*
3053                          * have to flush r2tqueue to avoid r2t leaks
3054                          */
3055                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3056                                 sizeof(void*))) {
3057                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3058                                         sizeof(void*));
3059                         }
3060                         if (conn->tmabort_state == TMABORT_INITIAL) {
3061                                 conn->tmfcmd_pdus_cnt++;
3062                                 conn->tmabort_timer.expires = 3*HZ + jiffies;
3063                                 conn->tmabort_timer.function =
3064                                                 iscsi_tmabort_timedout;
3065                                 conn->tmabort_timer.data = (unsigned long)ctask;
3066                                 add_timer(&conn->tmabort_timer);
3067                                 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3068                         } else {
3069                                 if (!ctask->sc ||
3070                                     conn->tmabort_state == TMABORT_SUCCESS) {
3071                                         conn->tmabort_state = TMABORT_INITIAL;
3072                                         spin_unlock_bh(&session->lock);
3073                                         up(&conn->xmitsema);
3074                                         goto success;
3075                                 }
3076                                 conn->tmabort_state = TMABORT_INITIAL;
3077                                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3078                         }
3079                         spin_unlock_bh(&session->lock);
3080                 }
3081         }
3082         up(&conn->xmitsema);
3083
3084
3085         /*
3086          * block eh thread until:
3087          *
3088          * 1) abort response;
3089          * 2) abort timeout;
3090          * 3) session re-opened;
3091          * 4) session terminated;
3092          */
3093         for (;;) {
3094                 int p_state = session->state;
3095
3096                 rc = wait_event_interruptible(conn->ehwait,
3097                         (p_state == ISCSI_STATE_LOGGED_IN ?
3098                          (session->state == ISCSI_STATE_TERMINATE ||
3099                           conn->tmabort_state != TMABORT_INITIAL) :
3100                          (session->state == ISCSI_STATE_TERMINATE ||
3101                           session->state == ISCSI_STATE_LOGGED_IN)));
3102                 if (rc) {
3103                         /* shutdown.. */
3104                         session->state = ISCSI_STATE_TERMINATE;
3105                         goto failed;
3106                 }
3107
3108                 if (signal_pending(current))
3109                         flush_signals(current);
3110
3111                 if (session->state == ISCSI_STATE_TERMINATE)
3112                         goto failed;
3113
3114                 spin_lock_bh(&session->lock);
3115                 if (sc->SCp.phase == session->age &&
3116                    (conn->tmabort_state == TMABORT_TIMEDOUT ||
3117                     conn->tmabort_state == TMABORT_FAILED)) {
3118                         conn->tmabort_state = TMABORT_INITIAL;
3119                         if (!ctask->sc) {
3120                                 /*
3121                                  * ctask completed before tmf abort response or
3122                                  * time out.
3123                                  * But session is still ok => Happy Retry.
3124                                  */
3125                                 spin_unlock_bh(&session->lock);
3126                                 break;
3127                         }
3128                         spin_unlock_bh(&session->lock);
3129                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3130                         continue;
3131                 }
3132                 spin_unlock_bh(&session->lock);
3133                 break;
3134         }
3135
3136 success:
3137         debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3138         rc = SUCCESS;
3139         goto exit;
3140
3141 failed:
3142         debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3143         rc = FAILED;
3144
3145 exit:
3146         del_timer_sync(&conn->tmabort_timer);
3147
3148         down(&conn->xmitsema);
3149         if (conn->sock) {
3150                 struct sock *sk = conn->sock->sk;
3151
3152                 write_lock_bh(&sk->sk_callback_lock);
3153                 iscsi_ctask_cleanup(conn, ctask);
3154                 write_unlock_bh(&sk->sk_callback_lock);
3155         }
3156         up(&conn->xmitsema);
3157         return rc;
3158 }
3159
3160 static int
3161 iscsi_r2tpool_alloc(struct iscsi_session *session)
3162 {
3163         int i;
3164         int cmd_i;
3165
3166         /*
3167          * initialize per-task: R2T pool and xmit queue
3168          */
3169         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3170                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3171
3172                 /*
3173                  * pre-allocated x4 as much r2ts to handle race when
3174                  * target acks DataOut faster than we data_xmit() queues
3175                  * could replenish r2tqueue.
3176                  */
3177
3178                 /* R2T pool */
3179                 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3180                         (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3181                         goto r2t_alloc_fail;
3182                 }
3183
3184                 /* R2T xmit queue */
3185                 ctask->r2tqueue = kfifo_alloc(
3186                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3187                 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3188                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3189                &nb