[SCSI] iscsi_tcp: remove unused r2t handling
[linux-2.6.git] / drivers / scsi / iscsi_tcp.c
1 /*
2  * iSCSI Initiator over TCP/IP Data-Path
3  *
4  * Copyright (C) 2004 Dmitry Yusupov
5  * Copyright (C) 2004 Alex Aizman
6  * Copyright (C) 2005 - 2006 Mike Christie
7  * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
8  * maintained by open-iscsi@googlegroups.com
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published
12  * by the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful, but
16  * WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18  * General Public License for more details.
19  *
20  * See the file COPYING included with this distribution for more details.
21  *
22  * Credits:
23  *      Christoph Hellwig
24  *      FUJITA Tomonori
25  *      Arne Redlich
26  *      Zhenyu Wang
27  */
28
29 #include <linux/types.h>
30 #include <linux/list.h>
31 #include <linux/inet.h>
32 #include <linux/file.h>
33 #include <linux/blkdev.h>
34 #include <linux/crypto.h>
35 #include <linux/delay.h>
36 #include <linux/kfifo.h>
37 #include <linux/scatterlist.h>
38 #include <net/tcp.h>
39 #include <scsi/scsi_cmnd.h>
40 #include <scsi/scsi_device.h>
41 #include <scsi/scsi_host.h>
42 #include <scsi/scsi.h>
43 #include <scsi/scsi_transport_iscsi.h>
44
45 #include "iscsi_tcp.h"
46
47 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
48               "Alex Aizman <itn780@yahoo.com>");
49 MODULE_DESCRIPTION("iSCSI/TCP data-path");
50 MODULE_LICENSE("GPL");
51 #undef DEBUG_TCP
52 #define DEBUG_ASSERT
53
54 #ifdef DEBUG_TCP
55 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
56 #else
57 #define debug_tcp(fmt...)
58 #endif
59
60 static struct scsi_transport_template *iscsi_tcp_scsi_transport;
61 static struct scsi_host_template iscsi_sht;
62 static struct iscsi_transport iscsi_tcp_transport;
63
64 static unsigned int iscsi_max_lun = 512;
65 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
66
67 static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
68                                    struct iscsi_segment *segment);
69
70 /*
71  * Scatterlist handling: inside the iscsi_segment, we
72  * remember an index into the scatterlist, and set data/size
73  * to the current scatterlist entry. For highmem pages, we
74  * kmap as needed.
75  *
76  * Note that the page is unmapped when we return from
77  * TCP's data_ready handler, so we may end up mapping and
78  * unmapping the same page repeatedly. The whole reason
79  * for this is that we shouldn't keep the page mapped
80  * outside the softirq.
81  */
82
83 /**
84  * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
85  * @segment: the buffer object
86  * @sg: scatterlist
87  * @offset: byte offset into that sg entry
88  *
89  * This function sets up the segment so that subsequent
90  * data is copied to the indicated sg entry, at the given
91  * offset.
92  */
93 static inline void
94 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,
95                           struct scatterlist *sg, unsigned int offset)
96 {
97         segment->sg = sg;
98         segment->sg_offset = offset;
99         segment->size = min(sg->length - offset,
100                             segment->total_size - segment->total_copied);
101         segment->data = NULL;
102 }
103
104 /**
105  * iscsi_tcp_segment_map - map the current S/G page
106  * @segment: iscsi_segment
107  * @recv: 1 if called from recv path
108  *
109  * We only need to possibly kmap data if scatter lists are being used,
110  * because the iscsi passthrough and internal IO paths will never use high
111  * mem pages.
112  */
113 static inline void
114 iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)
115 {
116         struct scatterlist *sg;
117
118         if (segment->data != NULL || !segment->sg)
119                 return;
120
121         sg = segment->sg;
122         BUG_ON(segment->sg_mapped);
123         BUG_ON(sg->length == 0);
124
125         /*
126          * If the page count is greater than one it is ok to send
127          * to the network layer's zero copy send path. If not we
128          * have to go the slow sendmsg path. We always map for the
129          * recv path.
130          */
131         if (page_count(sg_page(sg)) >= 1 && !recv)
132                 return;
133
134         debug_tcp("iscsi_tcp_segment_map %s %p\n", recv ? "recv" : "xmit",
135                   segment);
136         segment->sg_mapped = kmap_atomic(sg_page(sg), KM_SOFTIRQ0);
137         segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;
138 }
139
140 static inline void
141 iscsi_tcp_segment_unmap(struct iscsi_segment *segment)
142 {
143         debug_tcp("iscsi_tcp_segment_unmap %p\n", segment);
144
145         if (segment->sg_mapped) {
146                 debug_tcp("iscsi_tcp_segment_unmap valid\n");
147                 kunmap_atomic(segment->sg_mapped, KM_SOFTIRQ0);
148                 segment->sg_mapped = NULL;
149                 segment->data = NULL;
150         }
151 }
152
153 /*
154  * Splice the digest buffer into the buffer
155  */
156 static inline void
157 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
158 {
159         segment->data = digest;
160         segment->digest_len = ISCSI_DIGEST_SIZE;
161         segment->total_size += ISCSI_DIGEST_SIZE;
162         segment->size = ISCSI_DIGEST_SIZE;
163         segment->copied = 0;
164         segment->sg = NULL;
165         segment->hash = NULL;
166 }
167
168 /**
169  * iscsi_tcp_segment_done - check whether the segment is complete
170  * @segment: iscsi segment to check
171  * @recv: set to one of this is called from the recv path
172  * @copied: number of bytes copied
173  *
174  * Check if we're done receiving this segment. If the receive
175  * buffer is full but we expect more data, move on to the
176  * next entry in the scatterlist.
177  *
178  * If the amount of data we received isn't a multiple of 4,
179  * we will transparently receive the pad bytes, too.
180  *
181  * This function must be re-entrant.
182  */
183 static inline int
184 iscsi_tcp_segment_done(struct iscsi_segment *segment, int recv, unsigned copied)
185 {
186         static unsigned char padbuf[ISCSI_PAD_LEN];
187         struct scatterlist sg;
188         unsigned int pad;
189
190         debug_tcp("copied %u %u size %u %s\n", segment->copied, copied,
191                   segment->size, recv ? "recv" : "xmit");
192         if (segment->hash && copied) {
193                 /*
194                  * If a segment is kmapd we must unmap it before sending
195                  * to the crypto layer since that will try to kmap it again.
196                  */
197                 iscsi_tcp_segment_unmap(segment);
198
199                 if (!segment->data) {
200                         sg_init_table(&sg, 1);
201                         sg_set_page(&sg, sg_page(segment->sg), copied,
202                                     segment->copied + segment->sg_offset +
203                                                         segment->sg->offset);
204                 } else
205                         sg_init_one(&sg, segment->data + segment->copied,
206                                     copied);
207                 crypto_hash_update(segment->hash, &sg, copied);
208         }
209
210         segment->copied += copied;
211         if (segment->copied < segment->size) {
212                 iscsi_tcp_segment_map(segment, recv);
213                 return 0;
214         }
215
216         segment->total_copied += segment->copied;
217         segment->copied = 0;
218         segment->size = 0;
219
220         /* Unmap the current scatterlist page, if there is one. */
221         iscsi_tcp_segment_unmap(segment);
222
223         /* Do we have more scatterlist entries? */
224         debug_tcp("total copied %u total size %u\n", segment->total_copied,
225                    segment->total_size);
226         if (segment->total_copied < segment->total_size) {
227                 /* Proceed to the next entry in the scatterlist. */
228                 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),
229                                           0);
230                 iscsi_tcp_segment_map(segment, recv);
231                 BUG_ON(segment->size == 0);
232                 return 0;
233         }
234
235         /* Do we need to handle padding? */
236         pad = iscsi_padding(segment->total_copied);
237         if (pad != 0) {
238                 debug_tcp("consume %d pad bytes\n", pad);
239                 segment->total_size += pad;
240                 segment->size = pad;
241                 segment->data = padbuf;
242                 return 0;
243         }
244
245         /*
246          * Set us up for transferring the data digest. hdr digest
247          * is completely handled in hdr done function.
248          */
249         if (segment->hash) {
250                 crypto_hash_final(segment->hash, segment->digest);
251                 iscsi_tcp_segment_splice_digest(segment,
252                                  recv ? segment->recv_digest : segment->digest);
253                 return 0;
254         }
255
256         return 1;
257 }
258
259 /**
260  * iscsi_tcp_xmit_segment - transmit segment
261  * @tcp_conn: the iSCSI TCP connection
262  * @segment: the buffer to transmnit
263  *
264  * This function transmits as much of the buffer as
265  * the network layer will accept, and returns the number of
266  * bytes transmitted.
267  *
268  * If CRC hashing is enabled, the function will compute the
269  * hash as it goes. When the entire segment has been transmitted,
270  * it will retrieve the hash value and send it as well.
271  */
272 static int
273 iscsi_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
274                        struct iscsi_segment *segment)
275 {
276         struct socket *sk = tcp_conn->sock;
277         unsigned int copied = 0;
278         int r = 0;
279
280         while (!iscsi_tcp_segment_done(segment, 0, r)) {
281                 struct scatterlist *sg;
282                 unsigned int offset, copy;
283                 int flags = 0;
284
285                 r = 0;
286                 offset = segment->copied;
287                 copy = segment->size - offset;
288
289                 if (segment->total_copied + segment->size < segment->total_size)
290                         flags |= MSG_MORE;
291
292                 /* Use sendpage if we can; else fall back to sendmsg */
293                 if (!segment->data) {
294                         sg = segment->sg;
295                         offset += segment->sg_offset + sg->offset;
296                         r = tcp_conn->sendpage(sk, sg_page(sg), offset, copy,
297                                                flags);
298                 } else {
299                         struct msghdr msg = { .msg_flags = flags };
300                         struct kvec iov = {
301                                 .iov_base = segment->data + offset,
302                                 .iov_len = copy
303                         };
304
305                         r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
306                 }
307
308                 if (r < 0) {
309                         iscsi_tcp_segment_unmap(segment);
310                         if (copied || r == -EAGAIN)
311                                 break;
312                         return r;
313                 }
314                 copied += r;
315         }
316         return copied;
317 }
318
319 /**
320  * iscsi_tcp_segment_recv - copy data to segment
321  * @tcp_conn: the iSCSI TCP connection
322  * @segment: the buffer to copy to
323  * @ptr: data pointer
324  * @len: amount of data available
325  *
326  * This function copies up to @len bytes to the
327  * given buffer, and returns the number of bytes
328  * consumed, which can actually be less than @len.
329  *
330  * If hash digest is enabled, the function will update the
331  * hash while copying.
332  * Combining these two operations doesn't buy us a lot (yet),
333  * but in the future we could implement combined copy+crc,
334  * just way we do for network layer checksums.
335  */
336 static int
337 iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
338                        struct iscsi_segment *segment, const void *ptr,
339                        unsigned int len)
340 {
341         unsigned int copy = 0, copied = 0;
342
343         while (!iscsi_tcp_segment_done(segment, 1, copy)) {
344                 if (copied == len) {
345                         debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
346                                   len);
347                         break;
348                 }
349
350                 copy = min(len - copied, segment->size - segment->copied);
351                 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy);
352                 memcpy(segment->data + segment->copied, ptr + copied, copy);
353                 copied += copy;
354         }
355         return copied;
356 }
357
358 static inline void
359 iscsi_tcp_dgst_header(struct hash_desc *hash, const void *hdr, size_t hdrlen,
360                       unsigned char digest[ISCSI_DIGEST_SIZE])
361 {
362         struct scatterlist sg;
363
364         sg_init_one(&sg, hdr, hdrlen);
365         crypto_hash_digest(hash, &sg, hdrlen, digest);
366 }
367
368 static inline int
369 iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
370                       struct iscsi_segment *segment)
371 {
372         if (!segment->digest_len)
373                 return 1;
374
375         if (memcmp(segment->recv_digest, segment->digest,
376                    segment->digest_len)) {
377                 debug_scsi("digest mismatch\n");
378                 return 0;
379         }
380
381         return 1;
382 }
383
384 /*
385  * Helper function to set up segment buffer
386  */
387 static inline void
388 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,
389                      iscsi_segment_done_fn_t *done, struct hash_desc *hash)
390 {
391         memset(segment, 0, sizeof(*segment));
392         segment->total_size = size;
393         segment->done = done;
394
395         if (hash) {
396                 segment->hash = hash;
397                 crypto_hash_init(hash);
398         }
399 }
400
401 static inline void
402 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
403                           size_t size, iscsi_segment_done_fn_t *done,
404                           struct hash_desc *hash)
405 {
406         __iscsi_segment_init(segment, size, done, hash);
407         segment->data = data;
408         segment->size = size;
409 }
410
411 static inline int
412 iscsi_segment_seek_sg(struct iscsi_segment *segment,
413                       struct scatterlist *sg_list, unsigned int sg_count,
414                       unsigned int offset, size_t size,
415                       iscsi_segment_done_fn_t *done, struct hash_desc *hash)
416 {
417         struct scatterlist *sg;
418         unsigned int i;
419
420         debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
421                   offset, size);
422         __iscsi_segment_init(segment, size, done, hash);
423         for_each_sg(sg_list, sg, sg_count, i) {
424                 debug_scsi("sg %d, len %u offset %u\n", i, sg->length,
425                            sg->offset);
426                 if (offset < sg->length) {
427                         iscsi_tcp_segment_init_sg(segment, sg, offset);
428                         return 0;
429                 }
430                 offset -= sg->length;
431         }
432
433         return ISCSI_ERR_DATA_OFFSET;
434 }
435
436 /**
437  * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
438  * @tcp_conn: iscsi connection to prep for
439  *
440  * This function always passes NULL for the hash argument, because when this
441  * function is called we do not yet know the final size of the header and want
442  * to delay the digest processing until we know that.
443  */
444 static void
445 iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)
446 {
447         debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn,
448                   tcp_conn->iscsi_conn->hdrdgst_en ? ", digest enabled" : "");
449         iscsi_segment_init_linear(&tcp_conn->in.segment,
450                                 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),
451                                 iscsi_tcp_hdr_recv_done, NULL);
452 }
453
454 /*
455  * Handle incoming reply to any other type of command
456  */
457 static int
458 iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,
459                          struct iscsi_segment *segment)
460 {
461         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
462         int rc = 0;
463
464         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
465                 return ISCSI_ERR_DATA_DGST;
466
467         rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,
468                         conn->data, tcp_conn->in.datalen);
469         if (rc)
470                 return rc;
471
472         iscsi_tcp_hdr_recv_prep(tcp_conn);
473         return 0;
474 }
475
476 static void
477 iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
478 {
479         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
480         struct hash_desc *rx_hash = NULL;
481
482         if (conn->datadgst_en &
483             !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
484                 rx_hash = &tcp_conn->rx_hash;
485
486         iscsi_segment_init_linear(&tcp_conn->in.segment,
487                                 conn->data, tcp_conn->in.datalen,
488                                 iscsi_tcp_data_recv_done, rx_hash);
489 }
490
491 /*
492  * must be called with session lock
493  */
494 static void iscsi_tcp_cleanup_task(struct iscsi_task *task)
495 {
496         struct iscsi_tcp_task *tcp_task = task->dd_data;
497         struct iscsi_r2t_info *r2t;
498
499         /* nothing to do for mgmt or pending tasks */
500         if (!task->sc || task->state == ISCSI_TASK_PENDING)
501                 return;
502
503         /* flush task's r2t queues */
504         while (__kfifo_get(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {
505                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
506                             sizeof(void*));
507                 debug_scsi("iscsi_tcp_cleanup_task pending r2t dropped\n");
508         }
509
510         r2t = tcp_task->r2t;
511         if (r2t != NULL) {
512                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
513                             sizeof(void*));
514                 tcp_task->r2t = NULL;
515         }
516 }
517
518 /**
519  * iscsi_data_in - SCSI Data-In Response processing
520  * @conn: iscsi connection
521  * @task: scsi command task
522  **/
523 static int
524 iscsi_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
525 {
526         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
527         struct iscsi_tcp_task *tcp_task = task->dd_data;
528         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
529         int datasn = be32_to_cpu(rhdr->datasn);
530         unsigned total_in_length = scsi_in(task->sc)->length;
531
532         iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);
533         if (tcp_conn->in.datalen == 0)
534                 return 0;
535
536         if (tcp_task->exp_datasn != datasn) {
537                 debug_tcp("%s: task->exp_datasn(%d) != rhdr->datasn(%d)\n",
538                           __func__, tcp_task->exp_datasn, datasn);
539                 return ISCSI_ERR_DATASN;
540         }
541
542         tcp_task->exp_datasn++;
543
544         tcp_task->data_offset = be32_to_cpu(rhdr->offset);
545         if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {
546                 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
547                           __func__, tcp_task->data_offset,
548                           tcp_conn->in.datalen, total_in_length);
549                 return ISCSI_ERR_DATA_OFFSET;
550         }
551
552         conn->datain_pdus_cnt++;
553         return 0;
554 }
555
556 /**
557  * iscsi_r2t_rsp - iSCSI R2T Response processing
558  * @conn: iscsi connection
559  * @task: scsi command task
560  **/
561 static int
562 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task)
563 {
564         struct iscsi_session *session = conn->session;
565         struct iscsi_tcp_task *tcp_task = task->dd_data;
566         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
567         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
568         struct iscsi_r2t_info *r2t;
569         int r2tsn = be32_to_cpu(rhdr->r2tsn);
570         int rc;
571
572         if (tcp_conn->in.datalen) {
573                 iscsi_conn_printk(KERN_ERR, conn,
574                                   "invalid R2t with datalen %d\n",
575                                   tcp_conn->in.datalen);
576                 return ISCSI_ERR_DATALEN;
577         }
578
579         if (tcp_task->exp_datasn != r2tsn){
580                 debug_tcp("%s: task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
581                           __func__, tcp_task->exp_datasn, r2tsn);
582                 return ISCSI_ERR_R2TSN;
583         }
584
585         /* fill-in new R2T associated with the task */
586         iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
587
588         if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) {
589                 iscsi_conn_printk(KERN_INFO, conn,
590                                   "dropping R2T itt %d in recovery.\n",
591                                   task->itt);
592                 return 0;
593         }
594
595         rc = __kfifo_get(tcp_task->r2tpool.queue, (void*)&r2t, sizeof(void*));
596         if (!rc) {
597                 iscsi_conn_printk(KERN_ERR, conn, "Could not allocate R2T. "
598                                   "Target has sent more R2Ts than it "
599                                   "negotiated for or driver has has leaked.\n");
600                 return ISCSI_ERR_PROTO;
601         }
602
603         r2t->exp_statsn = rhdr->statsn;
604         r2t->data_length = be32_to_cpu(rhdr->data_length);
605         if (r2t->data_length == 0) {
606                 iscsi_conn_printk(KERN_ERR, conn,
607                                   "invalid R2T with zero data len\n");
608                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
609                             sizeof(void*));
610                 return ISCSI_ERR_DATALEN;
611         }
612
613         if (r2t->data_length > session->max_burst)
614                 debug_scsi("invalid R2T with data len %u and max burst %u."
615                            "Attempting to execute request.\n",
616                             r2t->data_length, session->max_burst);
617
618         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
619         if (r2t->data_offset + r2t->data_length > scsi_out(task->sc)->length) {
620                 iscsi_conn_printk(KERN_ERR, conn,
621                                   "invalid R2T with data len %u at offset %u "
622                                   "and total length %d\n", r2t->data_length,
623                                   r2t->data_offset, scsi_out(task->sc)->length);
624                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
625                             sizeof(void*));
626                 return ISCSI_ERR_DATALEN;
627         }
628
629         r2t->ttt = rhdr->ttt; /* no flip */
630         r2t->datasn = 0;
631         r2t->sent = 0;
632
633         tcp_task->exp_datasn = r2tsn + 1;
634         __kfifo_put(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));
635         conn->r2t_pdus_cnt++;
636
637         iscsi_requeue_task(task);
638         return 0;
639 }
640
641 /*
642  * Handle incoming reply to DataIn command
643  */
644 static int
645 iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,
646                           struct iscsi_segment *segment)
647 {
648         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
649         struct iscsi_hdr *hdr = tcp_conn->in.hdr;
650         int rc;
651
652         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
653                 return ISCSI_ERR_DATA_DGST;
654
655         /* check for non-exceptional status */
656         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
657                 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
658                 if (rc)
659                         return rc;
660         }
661
662         iscsi_tcp_hdr_recv_prep(tcp_conn);
663         return 0;
664 }
665
666 /**
667  * iscsi_tcp_hdr_dissect - process PDU header
668  * @conn: iSCSI connection
669  * @hdr: PDU header
670  *
671  * This function analyzes the header of the PDU received,
672  * and performs several sanity checks. If the PDU is accompanied
673  * by data, the receive buffer is set up to copy the incoming data
674  * to the correct location.
675  */
676 static int
677 iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
678 {
679         int rc = 0, opcode, ahslen;
680         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
681         struct iscsi_task *task;
682
683         /* verify PDU length */
684         tcp_conn->in.datalen = ntoh24(hdr->dlength);
685         if (tcp_conn->in.datalen > conn->max_recv_dlength) {
686                 iscsi_conn_printk(KERN_ERR, conn,
687                                   "iscsi_tcp: datalen %d > %d\n",
688                                   tcp_conn->in.datalen, conn->max_recv_dlength);
689                 return ISCSI_ERR_DATALEN;
690         }
691
692         /* Additional header segments. So far, we don't
693          * process additional headers.
694          */
695         ahslen = hdr->hlength << 2;
696
697         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
698         /* verify itt (itt encoding: age+cid+itt) */
699         rc = iscsi_verify_itt(conn, hdr->itt);
700         if (rc)
701                 return rc;
702
703         debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
704                   opcode, ahslen, tcp_conn->in.datalen);
705
706         switch(opcode) {
707         case ISCSI_OP_SCSI_DATA_IN:
708                 spin_lock(&conn->session->lock);
709                 task = iscsi_itt_to_ctask(conn, hdr->itt);
710                 if (!task)
711                         rc = ISCSI_ERR_BAD_ITT;
712                 else
713                         rc = iscsi_data_in(conn, task);
714                 if (rc) {
715                         spin_unlock(&conn->session->lock);
716                         break;
717                 }
718
719                 if (tcp_conn->in.datalen) {
720                         struct iscsi_tcp_task *tcp_task = task->dd_data;
721                         struct hash_desc *rx_hash = NULL;
722                         struct scsi_data_buffer *sdb = scsi_in(task->sc);
723
724                         /*
725                          * Setup copy of Data-In into the Scsi_Cmnd
726                          * Scatterlist case:
727                          * We set up the iscsi_segment to point to the next
728                          * scatterlist entry to copy to. As we go along,
729                          * we move on to the next scatterlist entry and
730                          * update the digest per-entry.
731                          */
732                         if (conn->datadgst_en &&
733                             !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
734                                 rx_hash = &tcp_conn->rx_hash;
735
736                         debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
737                                   "datalen=%d)\n", tcp_conn,
738                                   tcp_task->data_offset,
739                                   tcp_conn->in.datalen);
740                         rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,
741                                                    sdb->table.sgl,
742                                                    sdb->table.nents,
743                                                    tcp_task->data_offset,
744                                                    tcp_conn->in.datalen,
745                                                    iscsi_tcp_process_data_in,
746                                                    rx_hash);
747                         spin_unlock(&conn->session->lock);
748                         return rc;
749                 }
750                 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
751                 spin_unlock(&conn->session->lock);
752                 break;
753         case ISCSI_OP_SCSI_CMD_RSP:
754                 if (tcp_conn->in.datalen) {
755                         iscsi_tcp_data_recv_prep(tcp_conn);
756                         return 0;
757                 }
758                 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
759                 break;
760         case ISCSI_OP_R2T:
761                 spin_lock(&conn->session->lock);
762                 task = iscsi_itt_to_ctask(conn, hdr->itt);
763                 if (!task)
764                         rc = ISCSI_ERR_BAD_ITT;
765                 else if (ahslen)
766                         rc = ISCSI_ERR_AHSLEN;
767                 else if (task->sc->sc_data_direction == DMA_TO_DEVICE)
768                         rc = iscsi_r2t_rsp(conn, task);
769                 else
770                         rc = ISCSI_ERR_PROTO;
771                 spin_unlock(&conn->session->lock);
772                 break;
773         case ISCSI_OP_LOGIN_RSP:
774         case ISCSI_OP_TEXT_RSP:
775         case ISCSI_OP_REJECT:
776         case ISCSI_OP_ASYNC_EVENT:
777                 /*
778                  * It is possible that we could get a PDU with a buffer larger
779                  * than 8K, but there are no targets that currently do this.
780                  * For now we fail until we find a vendor that needs it
781                  */
782                 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {
783                         iscsi_conn_printk(KERN_ERR, conn,
784                                           "iscsi_tcp: received buffer of "
785                                           "len %u but conn buffer is only %u "
786                                           "(opcode %0x)\n",
787                                           tcp_conn->in.datalen,
788                                           ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
789                         rc = ISCSI_ERR_PROTO;
790                         break;
791                 }
792
793                 /* If there's data coming in with the response,
794                  * receive it to the connection's buffer.
795                  */
796                 if (tcp_conn->in.datalen) {
797                         iscsi_tcp_data_recv_prep(tcp_conn);
798                         return 0;
799                 }
800         /* fall through */
801         case ISCSI_OP_LOGOUT_RSP:
802         case ISCSI_OP_NOOP_IN:
803         case ISCSI_OP_SCSI_TMFUNC_RSP:
804                 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
805                 break;
806         default:
807                 rc = ISCSI_ERR_BAD_OPCODE;
808                 break;
809         }
810
811         if (rc == 0) {
812                 /* Anything that comes with data should have
813                  * been handled above. */
814                 if (tcp_conn->in.datalen)
815                         return ISCSI_ERR_PROTO;
816                 iscsi_tcp_hdr_recv_prep(tcp_conn);
817         }
818
819         return rc;
820 }
821
822 /**
823  * iscsi_tcp_hdr_recv_done - process PDU header
824  *
825  * This is the callback invoked when the PDU header has
826  * been received. If the header is followed by additional
827  * header segments, we go back for more data.
828  */
829 static int
830 iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
831                         struct iscsi_segment *segment)
832 {
833         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
834         struct iscsi_hdr *hdr;
835
836         /* Check if there are additional header segments
837          * *prior* to computing the digest, because we
838          * may need to go back to the caller for more.
839          */
840         hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;
841         if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {
842                 /* Bump the header length - the caller will
843                  * just loop around and get the AHS for us, and
844                  * call again. */
845                 unsigned int ahslen = hdr->hlength << 2;
846
847                 /* Make sure we don't overflow */
848                 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
849                         return ISCSI_ERR_AHSLEN;
850
851                 segment->total_size += ahslen;
852                 segment->size += ahslen;
853                 return 0;
854         }
855
856         /* We're done processing the header. See if we're doing
857          * header digests; if so, set up the recv_digest buffer
858          * and go back for more. */
859         if (conn->hdrdgst_en) {
860                 if (segment->digest_len == 0) {
861                         /*
862                          * Even if we offload the digest processing we
863                          * splice it in so we can increment the skb/segment
864                          * counters in preparation for the data segment.
865                          */
866                         iscsi_tcp_segment_splice_digest(segment,
867                                                         segment->recv_digest);
868                         return 0;
869                 }
870
871                 if (!(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) {
872                         iscsi_tcp_dgst_header(&tcp_conn->rx_hash, hdr,
873                                 segment->total_copied - ISCSI_DIGEST_SIZE,
874                                 segment->digest);
875
876                         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
877                                 return ISCSI_ERR_HDR_DGST;
878                 }
879         }
880
881         tcp_conn->in.hdr = hdr;
882         return iscsi_tcp_hdr_dissect(conn, hdr);
883 }
884
885 inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn)
886 {
887         return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;
888 }
889
890 enum {
891         ISCSI_TCP_SEGMENT_DONE,         /* curr seg has been processed */
892         ISCSI_TCP_SKB_DONE,             /* skb is out of data */
893         ISCSI_TCP_CONN_ERR,             /* iscsi layer has fired a conn err */
894         ISCSI_TCP_SUSPENDED,            /* conn is suspended */
895 };
896
897 /**
898  * iscsi_tcp_recv_skb - Process skb
899  * @conn: iscsi connection
900  * @skb: network buffer with header and/or data segment
901  * @offset: offset in skb
902  * @offload: bool indicating if transfer was offloaded
903  */
904 int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,
905                        unsigned int offset, bool offloaded, int *status)
906 {
907         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
908         struct iscsi_segment *segment = &tcp_conn->in.segment;
909         struct skb_seq_state seq;
910         unsigned int consumed = 0;
911         int rc = 0;
912
913         debug_tcp("in %d bytes\n", skb->len - offset);
914
915         if (unlikely(conn->suspend_rx)) {
916                 debug_tcp("conn %d Rx suspended!\n", conn->id);
917                 *status = ISCSI_TCP_SUSPENDED;
918                 return 0;
919         }
920
921         if (offloaded) {
922                 segment->total_copied = segment->total_size;
923                 goto segment_done;
924         }
925
926         skb_prepare_seq_read(skb, offset, skb->len, &seq);
927         while (1) {
928                 unsigned int avail;
929                 const u8 *ptr;
930
931                 avail = skb_seq_read(consumed, &ptr, &seq);
932                 if (avail == 0) {
933                         debug_tcp("no more data avail. Consumed %d\n",
934                                   consumed);
935                         *status = ISCSI_TCP_SKB_DONE;
936                         skb_abort_seq_read(&seq);
937                         goto skb_done;
938                 }
939                 BUG_ON(segment->copied >= segment->size);
940
941                 debug_tcp("skb %p ptr=%p avail=%u\n", skb, ptr, avail);
942                 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);
943                 BUG_ON(rc == 0);
944                 consumed += rc;
945
946                 if (segment->total_copied >= segment->total_size) {
947                         skb_abort_seq_read(&seq);
948                         goto segment_done;
949                 }
950         }
951
952 segment_done:
953         *status = ISCSI_TCP_SEGMENT_DONE;
954         debug_tcp("segment done\n");
955         rc = segment->done(tcp_conn, segment);
956         if (rc != 0) {
957                 *status = ISCSI_TCP_CONN_ERR;
958                 debug_tcp("Error receiving PDU, errno=%d\n", rc);
959                 iscsi_conn_failure(conn, rc);
960                 return 0;
961         }
962         /* The done() functions sets up the next segment. */
963
964 skb_done:
965         conn->rxdata_octets += consumed;
966         return consumed;
967 }
968 EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb);
969
970 /**
971  * iscsi_tcp_recv - TCP receive in sendfile fashion
972  * @rd_desc: read descriptor
973  * @skb: socket buffer
974  * @offset: offset in skb
975  * @len: skb->len - offset
976  **/
977 static int
978 iscsi_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
979                unsigned int offset, size_t len)
980 {
981         struct iscsi_conn *conn = rd_desc->arg.data;
982         unsigned int consumed, total_consumed = 0;
983         int status;
984
985         debug_tcp("in %d bytes\n", skb->len - offset);
986
987         do {
988                 status = 0;
989                 consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status);
990                 offset += consumed;
991                 total_consumed += consumed;
992         } while (consumed != 0 && status != ISCSI_TCP_SKB_DONE);
993
994         debug_tcp("read %d bytes status %d\n", skb->len - offset, status);
995         return total_consumed;
996 }
997
998 static void
999 iscsi_tcp_data_ready(struct sock *sk, int flag)
1000 {
1001         struct iscsi_conn *conn = sk->sk_user_data;
1002         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1003         read_descriptor_t rd_desc;
1004
1005         read_lock(&sk->sk_callback_lock);
1006
1007         /*
1008          * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
1009          * We set count to 1 because we want the network layer to
1010          * hand us all the skbs that are available. iscsi_tcp_recv
1011          * handled pdus that cross buffers or pdus that still need data.
1012          */
1013         rd_desc.arg.data = conn;
1014         rd_desc.count = 1;
1015         tcp_read_sock(sk, &rd_desc, iscsi_tcp_recv);
1016
1017         read_unlock(&sk->sk_callback_lock);
1018
1019         /* If we had to (atomically) map a highmem page,
1020          * unmap it now. */
1021         iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
1022 }
1023
1024 static void
1025 iscsi_tcp_state_change(struct sock *sk)
1026 {
1027         struct iscsi_tcp_conn *tcp_conn;
1028         struct iscsi_conn *conn;
1029         struct iscsi_session *session;
1030         void (*old_state_change)(struct sock *);
1031
1032         read_lock(&sk->sk_callback_lock);
1033
1034         conn = (struct iscsi_conn*)sk->sk_user_data;
1035         session = conn->session;
1036
1037         if ((sk->sk_state == TCP_CLOSE_WAIT ||
1038              sk->sk_state == TCP_CLOSE) &&
1039             !atomic_read(&sk->sk_rmem_alloc)) {
1040                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1041                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1042         }
1043
1044         tcp_conn = conn->dd_data;
1045         old_state_change = tcp_conn->old_state_change;
1046
1047         read_unlock(&sk->sk_callback_lock);
1048
1049         old_state_change(sk);
1050 }
1051
1052 /**
1053  * iscsi_write_space - Called when more output buffer space is available
1054  * @sk: socket space is available for
1055  **/
1056 static void
1057 iscsi_write_space(struct sock *sk)
1058 {
1059         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1060         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1061
1062         tcp_conn->old_write_space(sk);
1063         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1064         scsi_queue_work(conn->session->host, &conn->xmitwork);
1065 }
1066
1067 static void
1068 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1069 {
1070         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1071         struct sock *sk = tcp_conn->sock->sk;
1072
1073         /* assign new callbacks */
1074         write_lock_bh(&sk->sk_callback_lock);
1075         sk->sk_user_data = conn;
1076         tcp_conn->old_data_ready = sk->sk_data_ready;
1077         tcp_conn->old_state_change = sk->sk_state_change;
1078         tcp_conn->old_write_space = sk->sk_write_space;
1079         sk->sk_data_ready = iscsi_tcp_data_ready;
1080         sk->sk_state_change = iscsi_tcp_state_change;
1081         sk->sk_write_space = iscsi_write_space;
1082         write_unlock_bh(&sk->sk_callback_lock);
1083 }
1084
1085 static void
1086 iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
1087 {
1088         struct sock *sk = tcp_conn->sock->sk;
1089
1090         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1091         write_lock_bh(&sk->sk_callback_lock);
1092         sk->sk_user_data    = NULL;
1093         sk->sk_data_ready   = tcp_conn->old_data_ready;
1094         sk->sk_state_change = tcp_conn->old_state_change;
1095         sk->sk_write_space  = tcp_conn->old_write_space;
1096         sk->sk_no_check  = 0;
1097         write_unlock_bh(&sk->sk_callback_lock);
1098 }
1099
1100 /**
1101  * iscsi_xmit - TCP transmit
1102  **/
1103 static int
1104 iscsi_xmit(struct iscsi_conn *conn)
1105 {
1106         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1107         struct iscsi_segment *segment = &tcp_conn->out.segment;
1108         unsigned int consumed = 0;
1109         int rc = 0;
1110
1111         while (1) {
1112                 rc = iscsi_tcp_xmit_segment(tcp_conn, segment);
1113                 if (rc < 0) {
1114                         rc = ISCSI_ERR_XMIT_FAILED;
1115                         goto error;
1116                 }
1117                 if (rc == 0)
1118                         break;
1119
1120                 consumed += rc;
1121
1122                 if (segment->total_copied >= segment->total_size) {
1123                         if (segment->done != NULL) {
1124                                 rc = segment->done(tcp_conn, segment);
1125                                 if (rc != 0)
1126                                         goto error;
1127                         }
1128                 }
1129         }
1130
1131         debug_tcp("xmit %d bytes\n", consumed);
1132
1133         conn->txdata_octets += consumed;
1134         return consumed;
1135
1136 error:
1137         /* Transmit error. We could initiate error recovery
1138          * here. */
1139         debug_tcp("Error sending PDU, errno=%d\n", rc);
1140         iscsi_conn_failure(conn, rc);
1141         return -EIO;
1142 }
1143
1144 /**
1145  * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1146  */
1147 static inline int
1148 iscsi_tcp_xmit_qlen(struct iscsi_conn *conn)
1149 {
1150         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1151         struct iscsi_segment *segment = &tcp_conn->out.segment;
1152
1153         return segment->total_copied - segment->total_size;
1154 }
1155
1156 static int iscsi_tcp_flush(struct iscsi_task *task)
1157 {
1158         struct iscsi_conn *conn = task->conn;
1159         int rc;
1160
1161         while (iscsi_tcp_xmit_qlen(conn)) {
1162                 rc = iscsi_xmit(conn);
1163                 if (rc == 0)
1164                         return -EAGAIN;
1165                 if (rc < 0)
1166                         return rc;
1167         }
1168
1169         return 0;
1170 }
1171
1172 /*
1173  * This is called when we're done sending the header.
1174  * Simply copy the data_segment to the send segment, and return.
1175  */
1176 static int
1177 iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
1178                         struct iscsi_segment *segment)
1179 {
1180         tcp_conn->out.segment = tcp_conn->out.data_segment;
1181         debug_tcp("Header done. Next segment size %u total_size %u\n",
1182                   tcp_conn->out.segment.size, tcp_conn->out.segment.total_size);
1183         return 0;
1184 }
1185
1186 static void
1187 iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1188 {
1189         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1190
1191         debug_tcp("%s(%p%s)\n", __func__, tcp_conn,
1192                         conn->hdrdgst_en? ", digest enabled" : "");
1193
1194         /* Clear the data segment - needs to be filled in by the
1195          * caller using iscsi_tcp_send_data_prep() */
1196         memset(&tcp_conn->out.data_segment, 0, sizeof(struct iscsi_segment));
1197
1198         /* If header digest is enabled, compute the CRC and
1199          * place the digest into the same buffer. We make
1200          * sure that both iscsi_tcp_task and mtask have
1201          * sufficient room.
1202          */
1203         if (conn->hdrdgst_en) {
1204                 iscsi_tcp_dgst_header(&tcp_conn->tx_hash, hdr, hdrlen,
1205                                       hdr + hdrlen);
1206                 hdrlen += ISCSI_DIGEST_SIZE;
1207         }
1208
1209         /* Remember header pointer for later, when we need
1210          * to decide whether there's a payload to go along
1211          * with the header. */
1212         tcp_conn->out.hdr = hdr;
1213
1214         iscsi_segment_init_linear(&tcp_conn->out.segment, hdr, hdrlen,
1215                                 iscsi_tcp_send_hdr_done, NULL);
1216 }
1217
1218 /*
1219  * Prepare the send buffer for the payload data.
1220  * Padding and checksumming will all be taken care
1221  * of by the iscsi_segment routines.
1222  */
1223 static int
1224 iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1225                          unsigned int count, unsigned int offset,
1226                          unsigned int len)
1227 {
1228         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1229         struct hash_desc *tx_hash = NULL;
1230         unsigned int hdr_spec_len;
1231
1232         debug_tcp("%s(%p, offset=%d, datalen=%d%s)\n", __func__,
1233                         tcp_conn, offset, len,
1234                         conn->datadgst_en? ", digest enabled" : "");
1235
1236         /* Make sure the datalen matches what the caller
1237            said he would send. */
1238         hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength);
1239         WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1240
1241         if (conn->datadgst_en)
1242                 tx_hash = &tcp_conn->tx_hash;
1243
1244         return iscsi_segment_seek_sg(&tcp_conn->out.data_segment,
1245                                    sg, count, offset, len,
1246                                    NULL, tx_hash);
1247 }
1248
1249 static void
1250 iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data,
1251                                    size_t len)
1252 {
1253         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1254         struct hash_desc *tx_hash = NULL;
1255         unsigned int hdr_spec_len;
1256
1257         debug_tcp("%s(%p, datalen=%d%s)\n", __func__, tcp_conn, len,
1258                   conn->datadgst_en? ", digest enabled" : "");
1259
1260         /* Make sure the datalen matches what the caller
1261            said he would send. */
1262         hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength);
1263         WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1264
1265         if (conn->datadgst_en)
1266                 tx_hash = &tcp_conn->tx_hash;
1267
1268         iscsi_segment_init_linear(&tcp_conn->out.data_segment,
1269                                 data, len, NULL, tx_hash);
1270 }
1271
1272 static int iscsi_tcp_pdu_init(struct iscsi_task *task,
1273                               unsigned int offset, unsigned int count)
1274 {
1275         struct iscsi_conn *conn = task->conn;
1276         int err = 0;
1277
1278         iscsi_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
1279
1280         if (!count)
1281                 return 0;
1282
1283         if (!task->sc)
1284                 iscsi_tcp_send_linear_data_prepare(conn, task->data, count);
1285         else {
1286                 struct scsi_data_buffer *sdb = scsi_out(task->sc);
1287
1288                 err = iscsi_tcp_send_data_prep(conn, sdb->table.sgl,
1289                                                sdb->table.nents, offset, count);
1290         }
1291
1292         if (err) {
1293                 iscsi_conn_failure(conn, err);
1294                 return -EIO;
1295         }
1296         return 0;
1297 }
1298
1299 static int iscsi_tcp_pdu_alloc(struct iscsi_task *task)
1300 {
1301         struct iscsi_tcp_task *tcp_task = task->dd_data;
1302
1303         task->hdr = &tcp_task->hdr.hdrbuf;
1304         task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
1305         return 0;
1306 }
1307
1308 /**
1309  * iscsi_tcp_task - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1310  * @conn: iscsi connection
1311  * @task: scsi command task
1312  * @sc: scsi command
1313  **/
1314 static int iscsi_tcp_task_init(struct iscsi_task *task)
1315 {
1316         struct iscsi_tcp_task *tcp_task = task->dd_data;
1317         struct iscsi_conn *conn = task->conn;
1318         struct scsi_cmnd *sc = task->sc;
1319         int err;
1320
1321         if (!sc) {
1322                 /*
1323                  * mgmt tasks do not have a scatterlist since they come
1324                  * in from the iscsi interface.
1325                  */
1326                 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id,
1327                            task->itt);
1328
1329                 return conn->session->tt->init_pdu(task, 0, task->data_count);
1330         }
1331
1332         BUG_ON(__kfifo_len(tcp_task->r2tqueue));
1333         tcp_task->exp_datasn = 0;
1334
1335         /* Prepare PDU, optionally w/ immediate data */
1336         debug_scsi("task deq [cid %d itt 0x%x imm %d unsol %d]\n",
1337                     conn->id, task->itt, task->imm_count,
1338                     task->unsol_r2t.data_length);
1339
1340         err = conn->session->tt->init_pdu(task, 0, task->imm_count);
1341         if (err)
1342                 return err;
1343         task->imm_count = 0;
1344         return 0;
1345 }
1346
1347 static struct iscsi_r2t_info *iscsi_tcp_get_curr_r2t(struct iscsi_task *task)
1348 {
1349         struct iscsi_session *session = task->conn->session;
1350         struct iscsi_tcp_task *tcp_task = task->dd_data;
1351         struct iscsi_r2t_info *r2t = NULL;
1352
1353         if (iscsi_task_has_unsol_data(task))
1354                 r2t = &task->unsol_r2t;
1355         else {
1356                 spin_lock_bh(&session->lock);
1357                 if (tcp_task->r2t) {
1358                         r2t = tcp_task->r2t;
1359                         /* Continue with this R2T? */
1360                         if (r2t->data_length <= r2t->sent) {
1361                                 debug_scsi("  done with r2t %p\n", r2t);
1362                                 __kfifo_put(tcp_task->r2tpool.queue,
1363                                             (void *)&tcp_task->r2t,
1364                                             sizeof(void *));
1365                                 tcp_task->r2t = r2t = NULL;
1366                         }
1367                 }
1368
1369                 if (r2t == NULL) {
1370                         __kfifo_get(tcp_task->r2tqueue,
1371                                     (void *)&tcp_task->r2t, sizeof(void *));
1372                         r2t = tcp_task->r2t;
1373                 }
1374                 spin_unlock_bh(&session->lock);
1375         }
1376
1377         return r2t;
1378 }
1379
1380 /*
1381  * iscsi_tcp_task_xmit - xmit normal PDU task
1382  * @task: iscsi command task
1383  *
1384  * We're expected to return 0 when everything was transmitted succesfully,
1385  * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1386  * of error.
1387  */
1388 static int iscsi_tcp_task_xmit(struct iscsi_task *task)
1389 {
1390         struct iscsi_conn *conn = task->conn;
1391         struct iscsi_session *session = conn->session;
1392         struct iscsi_r2t_info *r2t;
1393         int rc = 0;
1394
1395 flush:
1396         /* Flush any pending data first. */
1397         rc = session->tt->xmit_pdu(task);
1398         if (rc < 0)
1399                 return rc;
1400
1401         /* mgmt command */
1402         if (!task->sc) {
1403                 if (task->hdr->itt == RESERVED_ITT)
1404                         iscsi_put_task(task);
1405                 return 0;
1406         }
1407
1408         /* Are we done already? */
1409         if (task->sc->sc_data_direction != DMA_TO_DEVICE)
1410                 return 0;
1411
1412         r2t = iscsi_tcp_get_curr_r2t(task);
1413         if (r2t == NULL) {
1414                 /* Waiting for more R2Ts to arrive. */
1415                 debug_tcp("no R2Ts yet\n");
1416                 return 0;
1417         }
1418
1419         rc = conn->session->tt->alloc_pdu(task);
1420         if (rc)
1421                 return rc;
1422         iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr);
1423
1424         debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1425                    r2t, r2t->datasn - 1, task->hdr->itt,
1426                    r2t->data_offset + r2t->sent, r2t->data_count);
1427
1428         rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent,
1429                                          r2t->data_count);
1430         if (rc)
1431                 return rc;
1432         r2t->sent += r2t->data_count;
1433         goto flush;
1434 }
1435
1436 static struct iscsi_cls_conn *
1437 iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1438 {
1439         struct iscsi_conn *conn;
1440         struct iscsi_cls_conn *cls_conn;
1441         struct iscsi_tcp_conn *tcp_conn;
1442
1443         cls_conn = iscsi_conn_setup(cls_session, sizeof(*tcp_conn), conn_idx);
1444         if (!cls_conn)
1445                 return NULL;
1446         conn = cls_conn->dd_data;
1447         /*
1448          * due to strange issues with iser these are not set
1449          * in iscsi_conn_setup
1450          */
1451         conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
1452
1453         tcp_conn = conn->dd_data;
1454         tcp_conn->iscsi_conn = conn;
1455
1456         tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1457                                                   CRYPTO_ALG_ASYNC);
1458         tcp_conn->tx_hash.flags = 0;
1459         if (IS_ERR(tcp_conn->tx_hash.tfm))
1460                 goto free_conn;
1461
1462         tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1463                                                   CRYPTO_ALG_ASYNC);
1464         tcp_conn->rx_hash.flags = 0;
1465         if (IS_ERR(tcp_conn->rx_hash.tfm))
1466                 goto free_tx_tfm;
1467
1468         return cls_conn;
1469
1470 free_tx_tfm:
1471         crypto_free_hash(tcp_conn->tx_hash.tfm);
1472 free_conn:
1473         iscsi_conn_printk(KERN_ERR, conn,
1474                           "Could not create connection due to crc32c "
1475                           "loading error. Make sure the crc32c "
1476                           "module is built as a module or into the "
1477                           "kernel\n");
1478         iscsi_conn_teardown(cls_conn);
1479         return NULL;
1480 }
1481
1482 static void
1483 iscsi_tcp_release_conn(struct iscsi_conn *conn)
1484 {
1485         struct iscsi_session *session = conn->session;
1486         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1487         struct socket *sock = tcp_conn->sock;
1488
1489         if (!sock)
1490                 return;
1491
1492         sock_hold(sock->sk);
1493         iscsi_conn_restore_callbacks(tcp_conn);
1494         sock_put(sock->sk);
1495
1496         spin_lock_bh(&session->lock);
1497         tcp_conn->sock = NULL;
1498         spin_unlock_bh(&session->lock);
1499         sockfd_put(sock);
1500 }
1501
1502 static void
1503 iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1504 {
1505         struct iscsi_conn *conn = cls_conn->dd_data;
1506         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1507
1508         iscsi_tcp_release_conn(conn);
1509
1510         if (tcp_conn->tx_hash.tfm)
1511                 crypto_free_hash(tcp_conn->tx_hash.tfm);
1512         if (tcp_conn->rx_hash.tfm)
1513                 crypto_free_hash(tcp_conn->rx_hash.tfm);
1514
1515         iscsi_conn_teardown(cls_conn);
1516 }
1517
1518 static void
1519 iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1520 {
1521         struct iscsi_conn *conn = cls_conn->dd_data;
1522         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1523
1524         /* userspace may have goofed up and not bound us */
1525         if (!tcp_conn->sock)
1526                 return;
1527         /*
1528          * Make sure our recv side is stopped.
1529          * Older tools called conn stop before ep_disconnect
1530          * so IO could still be coming in.
1531          */
1532         write_lock_bh(&tcp_conn->sock->sk->sk_callback_lock);
1533         set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
1534         write_unlock_bh(&tcp_conn->sock->sk->sk_callback_lock);
1535
1536         iscsi_conn_stop(cls_conn, flag);
1537         iscsi_tcp_release_conn(conn);
1538 }
1539
1540 static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
1541                               char *buf, int *port,
1542                               int (*getname)(struct socket *, struct sockaddr *,
1543                                         int *addrlen))
1544 {
1545         struct sockaddr_storage *addr;
1546         struct sockaddr_in6 *sin6;
1547         struct sockaddr_in *sin;
1548         int rc = 0, len;
1549
1550         addr = kmalloc(sizeof(*addr), GFP_KERNEL);
1551         if (!addr)
1552                 return -ENOMEM;
1553
1554         if (getname(sock, (struct sockaddr *) addr, &len)) {
1555                 rc = -ENODEV;
1556                 goto free_addr;
1557         }
1558
1559         switch (addr->ss_family) {
1560         case AF_INET:
1561                 sin = (struct sockaddr_in *)addr;
1562                 spin_lock_bh(&conn->session->lock);
1563                 sprintf(buf, "%pI4", &sin->sin_addr.s_addr);
1564                 *port = be16_to_cpu(sin->sin_port);
1565                 spin_unlock_bh(&conn->session->lock);
1566                 break;
1567         case AF_INET6:
1568                 sin6 = (struct sockaddr_in6 *)addr;
1569                 spin_lock_bh(&conn->session->lock);
1570                 sprintf(buf, "%pI6", &sin6->sin6_addr);
1571                 *port = be16_to_cpu(sin6->sin6_port);
1572                 spin_unlock_bh(&conn->session->lock);
1573                 break;
1574         }
1575 free_addr:
1576         kfree(addr);
1577         return rc;
1578 }
1579
1580 static int
1581 iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1582                     struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1583                     int is_leading)
1584 {
1585         struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1586         struct iscsi_host *ihost = shost_priv(shost);
1587         struct iscsi_conn *conn = cls_conn->dd_data;
1588         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1589         struct sock *sk;
1590         struct socket *sock;
1591         int err;
1592
1593         /* lookup for existing socket */
1594         sock = sockfd_lookup((int)transport_eph, &err);
1595         if (!sock) {
1596                 iscsi_conn_printk(KERN_ERR, conn,
1597                                   "sockfd_lookup failed %d\n", err);
1598                 return -EEXIST;
1599         }
1600         /*
1601          * copy these values now because if we drop the session
1602          * userspace may still want to query the values since we will
1603          * be using them for the reconnect
1604          */
1605         err = iscsi_tcp_get_addr(conn, sock, conn->portal_address,
1606                                  &conn->portal_port, kernel_getpeername);
1607         if (err)
1608                 goto free_socket;
1609
1610         err = iscsi_tcp_get_addr(conn, sock, ihost->local_address,
1611                                 &ihost->local_port, kernel_getsockname);
1612         if (err)
1613                 goto free_socket;
1614
1615         err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
1616         if (err)
1617                 goto free_socket;
1618
1619         /* bind iSCSI connection and socket */
1620         tcp_conn->sock = sock;
1621
1622         /* setup Socket parameters */
1623         sk = sock->sk;
1624         sk->sk_reuse = 1;
1625         sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
1626         sk->sk_allocation = GFP_ATOMIC;
1627
1628         iscsi_conn_set_callbacks(conn);
1629         tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
1630         /*
1631          * set receive state machine into initial state
1632          */
1633         iscsi_tcp_hdr_recv_prep(tcp_conn);
1634         return 0;
1635
1636 free_socket:
1637         sockfd_put(sock);
1638         return err;
1639 }
1640
1641 static int
1642 iscsi_r2tpool_alloc(struct iscsi_session *session)
1643 {
1644         int i;
1645         int cmd_i;
1646
1647         /*
1648          * initialize per-task: R2T pool and xmit queue
1649          */
1650         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
1651                 struct iscsi_task *task = session->cmds[cmd_i];
1652                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1653
1654                 /*
1655                  * pre-allocated x2 as much r2ts to handle race when
1656                  * target acks DataOut faster than we data_xmit() queues
1657                  * could replenish r2tqueue.
1658                  */
1659
1660                 /* R2T pool */
1661                 if (iscsi_pool_init(&tcp_task->r2tpool,
1662                                     session->max_r2t * 2, NULL,
1663                                     sizeof(struct iscsi_r2t_info))) {
1664                         goto r2t_alloc_fail;
1665                 }
1666
1667                 /* R2T xmit queue */
1668                 tcp_task->r2tqueue = kfifo_alloc(
1669                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
1670                 if (tcp_task->r2tqueue == ERR_PTR(-ENOMEM)) {
1671                         iscsi_pool_free(&tcp_task->r2tpool);
1672                         goto r2t_alloc_fail;
1673                 }
1674         }
1675
1676         return 0;
1677
1678 r2t_alloc_fail:
1679         for (i = 0; i < cmd_i; i++) {
1680                 struct iscsi_task *task = session->cmds[i];
1681                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1682
1683                 kfifo_free(tcp_task->r2tqueue);
1684                 iscsi_pool_free(&tcp_task->r2tpool);
1685         }
1686         return -ENOMEM;
1687 }
1688
1689 static void
1690 iscsi_r2tpool_free(struct iscsi_session *session)
1691 {
1692         int i;
1693
1694         for (i = 0; i < session->cmds_max; i++) {
1695                 struct iscsi_task *task = session->cmds[i];
1696                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1697
1698                 kfifo_free(tcp_task->r2tqueue);
1699                 iscsi_pool_free(&tcp_task->r2tpool);
1700         }
1701 }
1702
1703 static int
1704 iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1705                      char *buf, int buflen)
1706 {
1707         struct iscsi_conn *conn = cls_conn->dd_data;
1708         struct iscsi_session *session = conn->session;
1709         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1710         int value;
1711
1712         switch(param) {
1713         case ISCSI_PARAM_HDRDGST_EN:
1714                 iscsi_set_param(cls_conn, param, buf, buflen);
1715                 break;
1716         case ISCSI_PARAM_DATADGST_EN:
1717                 iscsi_set_param(cls_conn, param, buf, buflen);
1718                 tcp_conn->sendpage = conn->datadgst_en ?
1719                         sock_no_sendpage : tcp_conn->sock->ops->sendpage;
1720                 break;
1721         case ISCSI_PARAM_MAX_R2T:
1722                 sscanf(buf, "%d", &value);
1723                 if (value <= 0 || !is_power_of_2(value))
1724                         return -EINVAL;
1725                 if (session->max_r2t == value)
1726                         break;
1727                 iscsi_r2tpool_free(session);
1728                 iscsi_set_param(cls_conn, param, buf, buflen);
1729                 if (iscsi_r2tpool_alloc(session))
1730                         return -ENOMEM;
1731                 break;
1732         default:
1733                 return iscsi_set_param(cls_conn, param, buf, buflen);
1734         }
1735
1736         return 0;
1737 }
1738
1739 static int
1740 iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1741                          enum iscsi_param param, char *buf)
1742 {
1743         struct iscsi_conn *conn = cls_conn->dd_data;
1744         int len;
1745
1746         switch(param) {
1747         case ISCSI_PARAM_CONN_PORT:
1748                 spin_lock_bh(&conn->session->lock);
1749                 len = sprintf(buf, "%hu\n", conn->portal_port);
1750                 spin_unlock_bh(&conn->session->lock);
1751                 break;
1752         case ISCSI_PARAM_CONN_ADDRESS:
1753                 spin_lock_bh(&conn->session->lock);
1754                 len = sprintf(buf, "%s\n", conn->portal_address);
1755                 spin_unlock_bh(&conn->session->lock);
1756                 break;
1757         default:
1758                 return iscsi_conn_get_param(cls_conn, param, buf);
1759         }
1760
1761         return len;
1762 }
1763
1764 static void
1765 iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
1766 {
1767         struct iscsi_conn *conn = cls_conn->dd_data;
1768         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1769
1770         stats->txdata_octets = conn->txdata_octets;
1771         stats->rxdata_octets = conn->rxdata_octets;
1772         stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
1773         stats->dataout_pdus = conn->dataout_pdus_cnt;
1774         stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
1775         stats->datain_pdus = conn->datain_pdus_cnt;
1776         stats->r2t_pdus = conn->r2t_pdus_cnt;
1777         stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
1778         stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
1779         stats->custom_length = 3;
1780         strcpy(stats->custom[0].desc, "tx_sendpage_failures");
1781         stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
1782         strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
1783         stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
1784         strcpy(stats->custom[2].desc, "eh_abort_cnt");
1785         stats->custom[2].value = conn->eh_abort_cnt;
1786 }
1787
1788 static struct iscsi_cls_session *
1789 iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1790                          uint16_t qdepth, uint32_t initial_cmdsn,
1791                          uint32_t *hostno)
1792 {
1793         struct iscsi_cls_session *cls_session;
1794         struct iscsi_session *session;
1795         struct Scsi_Host *shost;
1796
1797         if (ep) {
1798                 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
1799                 return NULL;
1800         }
1801
1802         shost = iscsi_host_alloc(&iscsi_sht, 0, qdepth);
1803         if (!shost)
1804                 return NULL;
1805         shost->transportt = iscsi_tcp_scsi_transport;
1806         shost->max_lun = iscsi_max_lun;
1807         shost->max_id = 0;
1808         shost->max_channel = 0;
1809         shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
1810
1811         if (iscsi_host_add(shost, NULL))
1812                 goto free_host;
1813         *hostno = shost->host_no;
1814
1815         cls_session = iscsi_session_setup(&iscsi_tcp_transport, shost, cmds_max,
1816                                           sizeof(struct iscsi_tcp_task),
1817                                           initial_cmdsn, 0);
1818         if (!cls_session)
1819                 goto remove_host;
1820         session = cls_session->dd_data;
1821
1822         shost->can_queue = session->scsi_cmds_max;
1823         if (iscsi_r2tpool_alloc(session))
1824                 goto remove_session;
1825         return cls_session;
1826
1827 remove_session:
1828         iscsi_session_teardown(cls_session);
1829 remove_host:
1830         iscsi_host_remove(shost);
1831 free_host:
1832         iscsi_host_free(shost);
1833         return NULL;
1834 }
1835
1836 static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
1837 {
1838         struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1839
1840         iscsi_r2tpool_free(cls_session->dd_data);
1841         iscsi_session_teardown(cls_session);
1842
1843         iscsi_host_remove(shost);
1844         iscsi_host_free(shost);
1845 }
1846
1847 static int iscsi_tcp_slave_configure(struct scsi_device *sdev)
1848 {
1849         blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
1850         blk_queue_dma_alignment(sdev->request_queue, 0);
1851         return 0;
1852 }
1853
1854 static struct scsi_host_template iscsi_sht = {
1855         .module                 = THIS_MODULE,
1856         .name                   = "iSCSI Initiator over TCP/IP",
1857         .queuecommand           = iscsi_queuecommand,
1858         .change_queue_depth     = iscsi_change_queue_depth,
1859         .can_queue              = ISCSI_DEF_XMIT_CMDS_MAX - 1,
1860         .sg_tablesize           = 4096,
1861         .max_sectors            = 0xFFFF,
1862         .cmd_per_lun            = ISCSI_DEF_CMD_PER_LUN,
1863         .eh_abort_handler       = iscsi_eh_abort,
1864         .eh_device_reset_handler= iscsi_eh_device_reset,
1865         .eh_target_reset_handler= iscsi_eh_target_reset,
1866         .use_clustering         = DISABLE_CLUSTERING,
1867         .slave_configure        = iscsi_tcp_slave_configure,
1868         .proc_name              = "iscsi_tcp",
1869         .this_id                = -1,
1870 };
1871
1872 static struct iscsi_transport iscsi_tcp_transport = {
1873         .owner                  = THIS_MODULE,
1874         .name                   = "tcp",
1875         .caps                   = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
1876                                   | CAP_DATADGST,
1877         .param_mask             = ISCSI_MAX_RECV_DLENGTH |
1878                                   ISCSI_MAX_XMIT_DLENGTH |
1879                                   ISCSI_HDRDGST_EN |
1880                                   ISCSI_DATADGST_EN |
1881                                   ISCSI_INITIAL_R2T_EN |
1882                                   ISCSI_MAX_R2T |
1883                                   ISCSI_IMM_DATA_EN |
1884                                   ISCSI_FIRST_BURST |
1885                                   ISCSI_MAX_BURST |
1886                                   ISCSI_PDU_INORDER_EN |
1887                                   ISCSI_DATASEQ_INORDER_EN |
1888                                   ISCSI_ERL |
1889                                   ISCSI_CONN_PORT |
1890                                   ISCSI_CONN_ADDRESS |
1891                                   ISCSI_EXP_STATSN |
1892                                   ISCSI_PERSISTENT_PORT |
1893                                   ISCSI_PERSISTENT_ADDRESS |
1894                                   ISCSI_TARGET_NAME | ISCSI_TPGT |
1895                                   ISCSI_USERNAME | ISCSI_PASSWORD |
1896                                   ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
1897                                   ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
1898                                   ISCSI_LU_RESET_TMO |
1899                                   ISCSI_PING_TMO | ISCSI_RECV_TMO |
1900                                   ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
1901         .host_param_mask        = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
1902                                   ISCSI_HOST_INITIATOR_NAME |
1903                                   ISCSI_HOST_NETDEV_NAME,
1904         /* session management */
1905         .create_session         = iscsi_tcp_session_create,
1906         .destroy_session        = iscsi_tcp_session_destroy,
1907         /* connection management */
1908         .create_conn            = iscsi_tcp_conn_create,
1909         .bind_conn              = iscsi_tcp_conn_bind,
1910         .destroy_conn           = iscsi_tcp_conn_destroy,
1911         .set_param              = iscsi_conn_set_param,
1912         .get_conn_param         = iscsi_tcp_conn_get_param,
1913         .get_session_param      = iscsi_session_get_param,
1914         .start_conn             = iscsi_conn_start,
1915         .stop_conn              = iscsi_tcp_conn_stop,
1916         /* iscsi host params */
1917         .get_host_param         = iscsi_host_get_param,
1918         .set_host_param         = iscsi_host_set_param,
1919         /* IO */
1920         .send_pdu               = iscsi_conn_send_pdu,
1921         .get_stats              = iscsi_conn_get_stats,
1922         /* iscsi task/cmd helpers */
1923         .init_task              = iscsi_tcp_task_init,
1924         .xmit_task              = iscsi_tcp_task_xmit,
1925         .cleanup_task           = iscsi_tcp_cleanup_task,
1926         /* low level pdu helpers */
1927         .xmit_pdu               = iscsi_tcp_flush,
1928         .init_pdu               = iscsi_tcp_pdu_init,
1929         .alloc_pdu              = iscsi_tcp_pdu_alloc,
1930         /* recovery */
1931         .session_recovery_timedout = iscsi_session_recovery_timedout,
1932 };
1933
1934 static int __init
1935 iscsi_tcp_init(void)
1936 {
1937         if (iscsi_max_lun < 1) {
1938                 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
1939                        iscsi_max_lun);
1940                 return -EINVAL;
1941         }
1942
1943         iscsi_tcp_scsi_transport = iscsi_register_transport(
1944                                                         &iscsi_tcp_transport);
1945         if (!iscsi_tcp_scsi_transport)
1946                 return -ENODEV;
1947
1948         return 0;
1949 }
1950
1951 static void __exit
1952 iscsi_tcp_exit(void)
1953 {
1954         iscsi_unregister_transport(&iscsi_tcp_transport);
1955 }
1956
1957 module_init(iscsi_tcp_init);
1958 module_exit(iscsi_tcp_exit);