[SCSI] iscsi class: sysfs group is_visible callout for session attrs
[linux-2.6.git] / drivers / scsi / cxgbi / cxgb3i / cxgb3i.c
1 /*
2  * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
3  *
4  * Copyright (C) 2003-2008 Chelsio Communications.  All rights reserved.
5  *
6  * This program is distributed in the hope that it will be useful, but WITHOUT
7  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
9  * release for licensing terms and conditions.
10  *
11  * Written by:  Dimitris Michailidis (dm@chelsio.com)
12  *              Karen Xie (kxie@chelsio.com)
13  */
14
15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16
17 #include <linux/version.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <scsi/scsi_host.h>
21
22 #include "common.h"
23 #include "t3_cpl.h"
24 #include "t3cdev.h"
25 #include "cxgb3_defs.h"
26 #include "cxgb3_ctl_defs.h"
27 #include "cxgb3_offload.h"
28 #include "firmware_exports.h"
29 #include "cxgb3i.h"
30
31 static unsigned int dbg_level;
32 #include "../libcxgbi.h"
33
34 #define DRV_MODULE_NAME         "cxgb3i"
35 #define DRV_MODULE_DESC         "Chelsio T3 iSCSI Driver"
36 #define DRV_MODULE_VERSION      "2.0.0"
37 #define DRV_MODULE_RELDATE      "Jun. 2010"
38
39 static char version[] =
40         DRV_MODULE_DESC " " DRV_MODULE_NAME
41         " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
42
43 MODULE_AUTHOR("Chelsio Communications, Inc.");
44 MODULE_DESCRIPTION(DRV_MODULE_DESC);
45 MODULE_VERSION(DRV_MODULE_VERSION);
46 MODULE_LICENSE("GPL");
47
48 module_param(dbg_level, uint, 0644);
49 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
50
51 static int cxgb3i_rcv_win = 256 * 1024;
52 module_param(cxgb3i_rcv_win, int, 0644);
53 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
54
55 static int cxgb3i_snd_win = 128 * 1024;
56 module_param(cxgb3i_snd_win, int, 0644);
57 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
58
59 static int cxgb3i_rx_credit_thres = 10 * 1024;
60 module_param(cxgb3i_rx_credit_thres, int, 0644);
61 MODULE_PARM_DESC(rx_credit_thres,
62                  "RX credits return threshold in bytes (default=10KB)");
63
64 static unsigned int cxgb3i_max_connect = 8 * 1024;
65 module_param(cxgb3i_max_connect, uint, 0644);
66 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
67
68 static unsigned int cxgb3i_sport_base = 20000;
69 module_param(cxgb3i_sport_base, uint, 0644);
70 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
71
72 static void cxgb3i_dev_open(struct t3cdev *);
73 static void cxgb3i_dev_close(struct t3cdev *);
74 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
75
76 static struct cxgb3_client t3_client = {
77         .name = DRV_MODULE_NAME,
78         .handlers = cxgb3i_cpl_handlers,
79         .add = cxgb3i_dev_open,
80         .remove = cxgb3i_dev_close,
81         .event_handler = cxgb3i_dev_event_handler,
82 };
83
84 static struct scsi_host_template cxgb3i_host_template = {
85         .module         = THIS_MODULE,
86         .name           = DRV_MODULE_NAME,
87         .proc_name      = DRV_MODULE_NAME,
88         .can_queue      = CXGB3I_SCSI_HOST_QDEPTH,
89         .queuecommand   = iscsi_queuecommand,
90         .change_queue_depth = iscsi_change_queue_depth,
91         .sg_tablesize   = SG_ALL,
92         .max_sectors    = 0xFFFF,
93         .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
94         .eh_abort_handler = iscsi_eh_abort,
95         .eh_device_reset_handler = iscsi_eh_device_reset,
96         .eh_target_reset_handler = iscsi_eh_recover_target,
97         .target_alloc   = iscsi_target_alloc,
98         .use_clustering = DISABLE_CLUSTERING,
99         .this_id        = -1,
100 };
101
102 static struct iscsi_transport cxgb3i_iscsi_transport = {
103         .owner          = THIS_MODULE,
104         .name           = DRV_MODULE_NAME,
105         /* owner and name should be set already */
106         .caps           = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
107                                 | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
108                                 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
109         .host_param_mask        = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
110                                 ISCSI_HOST_INITIATOR_NAME |
111                                 ISCSI_HOST_NETDEV_NAME,
112         .attr_is_visible        = cxgbi_attr_is_visible,
113         .get_host_param = cxgbi_get_host_param,
114         .set_host_param = cxgbi_set_host_param,
115         /* session management */
116         .create_session = cxgbi_create_session,
117         .destroy_session        = cxgbi_destroy_session,
118         .get_session_param = iscsi_session_get_param,
119         /* connection management */
120         .create_conn    = cxgbi_create_conn,
121         .bind_conn      = cxgbi_bind_conn,
122         .destroy_conn   = iscsi_tcp_conn_teardown,
123         .start_conn     = iscsi_conn_start,
124         .stop_conn      = iscsi_conn_stop,
125         .get_conn_param = iscsi_conn_get_param,
126         .set_param      = cxgbi_set_conn_param,
127         .get_stats      = cxgbi_get_conn_stats,
128         /* pdu xmit req from user space */
129         .send_pdu       = iscsi_conn_send_pdu,
130         /* task */
131         .init_task      = iscsi_tcp_task_init,
132         .xmit_task      = iscsi_tcp_task_xmit,
133         .cleanup_task   = cxgbi_cleanup_task,
134         /* pdu */
135         .alloc_pdu      = cxgbi_conn_alloc_pdu,
136         .init_pdu       = cxgbi_conn_init_pdu,
137         .xmit_pdu       = cxgbi_conn_xmit_pdu,
138         .parse_pdu_itt  = cxgbi_parse_pdu_itt,
139         /* TCP connect/disconnect */
140         .get_ep_param   = cxgbi_get_ep_param,
141         .ep_connect     = cxgbi_ep_connect,
142         .ep_poll        = cxgbi_ep_poll,
143         .ep_disconnect  = cxgbi_ep_disconnect,
144         /* Error recovery timeout call */
145         .session_recovery_timedout = iscsi_session_recovery_timedout,
146 };
147
148 static struct scsi_transport_template *cxgb3i_stt;
149
150 /*
151  * CPL (Chelsio Protocol Language) defines a message passing interface between
152  * the host driver and Chelsio asic.
153  * The section below implments CPLs that related to iscsi tcp connection
154  * open/close/abort and data send/receive.
155  */
156
157 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
158
159 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
160                               const struct l2t_entry *e)
161 {
162         unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win);
163         struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
164
165         skb->priority = CPL_PRIORITY_SETUP;
166
167         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
168         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
169         req->local_port = csk->saddr.sin_port;
170         req->peer_port = csk->daddr.sin_port;
171         req->local_ip = csk->saddr.sin_addr.s_addr;
172         req->peer_ip = csk->daddr.sin_addr.s_addr;
173
174         req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
175                         V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
176                         V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
177         req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
178                         V_RCV_BUFSIZ(cxgb3i_rcv_win>>10));
179
180         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
181                 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
182                 csk, csk->state, csk->flags, csk->atid,
183                 &req->local_ip, ntohs(req->local_port),
184                 &req->peer_ip, ntohs(req->peer_port),
185                 csk->mss_idx, e->idx, e->smt_idx);
186
187         l2t_send(csk->cdev->lldev, skb, csk->l2t);
188 }
189
190 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
191 {
192         cxgbi_sock_act_open_req_arp_failure(NULL, skb);
193 }
194
195 /*
196  * CPL connection close request: host ->
197  *
198  * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
199  * the write queue (i.e., after any unsent txt data).
200  */
201 static void send_close_req(struct cxgbi_sock *csk)
202 {
203         struct sk_buff *skb = csk->cpl_close;
204         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
205         unsigned int tid = csk->tid;
206
207         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
208                 "csk 0x%p,%u,0x%lx,%u.\n",
209                 csk, csk->state, csk->flags, csk->tid);
210
211         csk->cpl_close = NULL;
212         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
213         req->wr.wr_lo = htonl(V_WR_TID(tid));
214         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
215         req->rsvd = htonl(csk->write_seq);
216
217         cxgbi_sock_skb_entail(csk, skb);
218         if (csk->state >= CTP_ESTABLISHED)
219                 push_tx_frames(csk, 1);
220 }
221
222 /*
223  * CPL connection abort request: host ->
224  *
225  * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
226  * for the same connection and also that we do not try to send a message
227  * after the connection has closed.
228  */
229 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
230 {
231         struct cpl_abort_req *req = cplhdr(skb);
232
233         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
234                 "t3dev 0x%p, tid %u, skb 0x%p.\n",
235                 tdev, GET_TID(req), skb);
236         req->cmd = CPL_ABORT_NO_RST;
237         cxgb3_ofld_send(tdev, skb);
238 }
239
240 static void send_abort_req(struct cxgbi_sock *csk)
241 {
242         struct sk_buff *skb = csk->cpl_abort_req;
243         struct cpl_abort_req *req;
244
245         if (unlikely(csk->state == CTP_ABORTING || !skb))
246                 return;
247         cxgbi_sock_set_state(csk, CTP_ABORTING);
248         cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
249         /* Purge the send queue so we don't send anything after an abort. */
250         cxgbi_sock_purge_write_queue(csk);
251
252         csk->cpl_abort_req = NULL;
253         req = (struct cpl_abort_req *)skb->head;
254         skb->priority = CPL_PRIORITY_DATA;
255         set_arp_failure_handler(skb, abort_arp_failure);
256         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
257         req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
258         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
259         req->rsvd0 = htonl(csk->snd_nxt);
260         req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
261         req->cmd = CPL_ABORT_SEND_RST;
262
263         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
264                 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
265                 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
266                 req->rsvd1);
267
268         l2t_send(csk->cdev->lldev, skb, csk->l2t);
269 }
270
271 /*
272  * CPL connection abort reply: host ->
273  *
274  * Send an ABORT_RPL message in response of the ABORT_REQ received.
275  */
276 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
277 {
278         struct sk_buff *skb = csk->cpl_abort_rpl;
279         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
280
281         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
282                 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
283                 csk, csk->state, csk->flags, csk->tid, rst_status);
284
285         csk->cpl_abort_rpl = NULL;
286         skb->priority = CPL_PRIORITY_DATA;
287         rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
288         rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
289         OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
290         rpl->cmd = rst_status;
291         cxgb3_ofld_send(csk->cdev->lldev, skb);
292 }
293
294 /*
295  * CPL connection rx data ack: host ->
296  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
297  * credits sent.
298  */
299 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
300 {
301         struct sk_buff *skb;
302         struct cpl_rx_data_ack *req;
303         u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
304
305         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
306                 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
307                 csk, csk->state, csk->flags, csk->tid, credits, dack);
308
309         skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
310         if (!skb) {
311                 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
312                 return 0;
313         }
314         req = (struct cpl_rx_data_ack *)skb->head;
315         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
316         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
317         req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
318                                 V_RX_CREDITS(credits));
319         skb->priority = CPL_PRIORITY_ACK;
320         cxgb3_ofld_send(csk->cdev->lldev, skb);
321         return credits;
322 }
323
324 /*
325  * CPL connection tx data: host ->
326  *
327  * Send iscsi PDU via TX_DATA CPL message. Returns the number of
328  * credits sent.
329  * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
330  * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
331  */
332
333 static unsigned int wrlen __read_mostly;
334 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
335
336 static void init_wr_tab(unsigned int wr_len)
337 {
338         int i;
339
340         if (skb_wrs[1])         /* already initialized */
341                 return;
342         for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
343                 int sgl_len = (3 * i) / 2 + (i & 1);
344
345                 sgl_len += 3;
346                 skb_wrs[i] = (sgl_len <= wr_len
347                               ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
348         }
349         wrlen = wr_len * 8;
350 }
351
352 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
353                                    int len, int req_completion)
354 {
355         struct tx_data_wr *req;
356         struct l2t_entry *l2t = csk->l2t;
357
358         skb_reset_transport_header(skb);
359         req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
360         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
361                         (req_completion ? F_WR_COMPL : 0));
362         req->wr_lo = htonl(V_WR_TID(csk->tid));
363         /* len includes the length of any HW ULP additions */
364         req->len = htonl(len);
365         /* V_TX_ULP_SUBMODE sets both the mode and submode */
366         req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
367                            V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
368         req->sndseq = htonl(csk->snd_nxt);
369         req->param = htonl(V_TX_PORT(l2t->smt_idx));
370
371         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
372                 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
373                                     V_TX_CPU_IDX(csk->rss_qid));
374                 /* sendbuffer is in units of 32KB. */
375                 req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15));
376                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
377         }
378 }
379
380 /**
381  * push_tx_frames -- start transmit
382  * @c3cn: the offloaded connection
383  * @req_completion: request wr_ack or not
384  *
385  * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
386  * connection's send queue and sends them on to T3.  Must be called with the
387  * connection's lock held.  Returns the amount of send buffer space that was
388  * freed as a result of sending queued data to T3.
389  */
390
391 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
392 {
393         kfree_skb(skb);
394 }
395
396 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
397 {
398         int total_size = 0;
399         struct sk_buff *skb;
400
401         if (unlikely(csk->state < CTP_ESTABLISHED ||
402                 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
403                         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
404                                 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
405                                 csk, csk->state, csk->flags, csk->tid);
406                 return 0;
407         }
408
409         while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
410                 int len = skb->len;     /* length before skb_push */
411                 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
412                 int wrs_needed = skb_wrs[frags];
413
414                 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
415                         wrs_needed = 1;
416
417                 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
418
419                 if (csk->wr_cred < wrs_needed) {
420                         log_debug(1 << CXGBI_DBG_PDU_TX,
421                                 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
422                                 csk, skb->len, skb->data_len, frags,
423                                 wrs_needed, csk->wr_cred);
424                         break;
425                 }
426
427                 __skb_unlink(skb, &csk->write_queue);
428                 skb->priority = CPL_PRIORITY_DATA;
429                 skb->csum = wrs_needed; /* remember this until the WR_ACK */
430                 csk->wr_cred -= wrs_needed;
431                 csk->wr_una_cred += wrs_needed;
432                 cxgbi_sock_enqueue_wr(csk, skb);
433
434                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
435                         "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
436                         "left %u, unack %u.\n",
437                         csk, skb->len, skb->data_len, frags, skb->csum,
438                         csk->wr_cred, csk->wr_una_cred);
439
440                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
441                         if ((req_completion &&
442                                 csk->wr_una_cred == wrs_needed) ||
443                              csk->wr_una_cred >= csk->wr_max_cred / 2) {
444                                 req_completion = 1;
445                                 csk->wr_una_cred = 0;
446                         }
447                         len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
448                         make_tx_data_wr(csk, skb, len, req_completion);
449                         csk->snd_nxt += len;
450                         cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
451                 }
452                 total_size += skb->truesize;
453                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
454                         "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
455                         csk, csk->tid, skb);
456                 set_arp_failure_handler(skb, arp_failure_skb_discard);
457                 l2t_send(csk->cdev->lldev, skb, csk->l2t);
458         }
459         return total_size;
460 }
461
462 /*
463  * Process a CPL_ACT_ESTABLISH message: -> host
464  * Updates connection state from an active establish CPL message.  Runs with
465  * the connection lock held.
466  */
467
468 static inline void free_atid(struct cxgbi_sock *csk)
469 {
470         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
471                 cxgb3_free_atid(csk->cdev->lldev, csk->atid);
472                 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
473                 cxgbi_sock_put(csk);
474         }
475 }
476
477 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
478 {
479         struct cxgbi_sock *csk = ctx;
480         struct cpl_act_establish *req = cplhdr(skb);
481         unsigned int tid = GET_TID(req);
482         unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
483         u32 rcv_isn = ntohl(req->rcv_isn);      /* real RCV_ISN + 1 */
484
485         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
486                 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
487                 atid, atid, csk, csk->state, csk->flags, rcv_isn);
488
489         cxgbi_sock_get(csk);
490         cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
491         csk->tid = tid;
492         cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
493
494         free_atid(csk);
495
496         csk->rss_qid = G_QNUM(ntohs(skb->csum));
497
498         spin_lock_bh(&csk->lock);
499         if (csk->retry_timer.function) {
500                 del_timer(&csk->retry_timer);
501                 csk->retry_timer.function = NULL;
502         }
503
504         if (unlikely(csk->state != CTP_ACTIVE_OPEN))
505                 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
506                         csk, csk->state, csk->flags, csk->tid);
507
508         csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
509         if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10))
510                 csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10);
511
512         cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
513
514         if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
515                 /* upper layer has requested closing */
516                 send_abort_req(csk);
517         else {
518                 if (skb_queue_len(&csk->write_queue))
519                         push_tx_frames(csk, 1);
520                 cxgbi_conn_tx_open(csk);
521         }
522
523         spin_unlock_bh(&csk->lock);
524         __kfree_skb(skb);
525         return 0;
526 }
527
528 /*
529  * Process a CPL_ACT_OPEN_RPL message: -> host
530  * Handle active open failures.
531  */
532 static int act_open_rpl_status_to_errno(int status)
533 {
534         switch (status) {
535         case CPL_ERR_CONN_RESET:
536                 return -ECONNREFUSED;
537         case CPL_ERR_ARP_MISS:
538                 return -EHOSTUNREACH;
539         case CPL_ERR_CONN_TIMEDOUT:
540                 return -ETIMEDOUT;
541         case CPL_ERR_TCAM_FULL:
542                 return -ENOMEM;
543         case CPL_ERR_CONN_EXIST:
544                 return -EADDRINUSE;
545         default:
546                 return -EIO;
547         }
548 }
549
550 static void act_open_retry_timer(unsigned long data)
551 {
552         struct sk_buff *skb;
553         struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
554
555         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
556                 "csk 0x%p,%u,0x%lx,%u.\n",
557                 csk, csk->state, csk->flags, csk->tid);
558
559         cxgbi_sock_get(csk);
560         spin_lock_bh(&csk->lock);
561         skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
562         if (!skb)
563                 cxgbi_sock_fail_act_open(csk, -ENOMEM);
564         else {
565                 skb->sk = (struct sock *)csk;
566                 set_arp_failure_handler(skb, act_open_arp_failure);
567                 send_act_open_req(csk, skb, csk->l2t);
568         }
569         spin_unlock_bh(&csk->lock);
570         cxgbi_sock_put(csk);
571 }
572
573 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
574 {
575         struct cxgbi_sock *csk = ctx;
576         struct cpl_act_open_rpl *rpl = cplhdr(skb);
577
578         pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
579                 csk, csk->state, csk->flags, csk->atid, rpl->status,
580                 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
581                 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
582
583         if (rpl->status != CPL_ERR_TCAM_FULL &&
584             rpl->status != CPL_ERR_CONN_EXIST &&
585             rpl->status != CPL_ERR_ARP_MISS)
586                 cxgb3_queue_tid_release(tdev, GET_TID(rpl));
587
588         cxgbi_sock_get(csk);
589         spin_lock_bh(&csk->lock);
590         if (rpl->status == CPL_ERR_CONN_EXIST &&
591             csk->retry_timer.function != act_open_retry_timer) {
592                 csk->retry_timer.function = act_open_retry_timer;
593                 mod_timer(&csk->retry_timer, jiffies + HZ / 2);
594         } else
595                 cxgbi_sock_fail_act_open(csk,
596                                 act_open_rpl_status_to_errno(rpl->status));
597
598         spin_unlock_bh(&csk->lock);
599         cxgbi_sock_put(csk);
600         __kfree_skb(skb);
601         return 0;
602 }
603
604 /*
605  * Process PEER_CLOSE CPL messages: -> host
606  * Handle peer FIN.
607  */
608 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
609 {
610         struct cxgbi_sock *csk = ctx;
611
612         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
613                 "csk 0x%p,%u,0x%lx,%u.\n",
614                 csk, csk->state, csk->flags, csk->tid);
615
616         cxgbi_sock_rcv_peer_close(csk);
617         __kfree_skb(skb);
618         return 0;
619 }
620
621 /*
622  * Process CLOSE_CONN_RPL CPL message: -> host
623  * Process a peer ACK to our FIN.
624  */
625 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
626                             void *ctx)
627 {
628         struct cxgbi_sock *csk = ctx;
629         struct cpl_close_con_rpl *rpl = cplhdr(skb);
630
631         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
632                 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
633                 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
634
635         cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
636         __kfree_skb(skb);
637         return 0;
638 }
639
640 /*
641  * Process ABORT_REQ_RSS CPL message: -> host
642  * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
643  * request except that we need to reply to it.
644  */
645
646 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
647                                  int *need_rst)
648 {
649         switch (abort_reason) {
650         case CPL_ERR_BAD_SYN: /* fall through */
651         case CPL_ERR_CONN_RESET:
652                 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
653         case CPL_ERR_XMIT_TIMEDOUT:
654         case CPL_ERR_PERSIST_TIMEDOUT:
655         case CPL_ERR_FINWAIT2_TIMEDOUT:
656         case CPL_ERR_KEEPALIVE_TIMEDOUT:
657                 return -ETIMEDOUT;
658         default:
659                 return -EIO;
660         }
661 }
662
663 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
664 {
665         const struct cpl_abort_req_rss *req = cplhdr(skb);
666         struct cxgbi_sock *csk = ctx;
667         int rst_status = CPL_ABORT_NO_RST;
668
669         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
670                 "csk 0x%p,%u,0x%lx,%u.\n",
671                 csk, csk->state, csk->flags, csk->tid);
672
673         if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
674             req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
675                 goto done;
676         }
677
678         cxgbi_sock_get(csk);
679         spin_lock_bh(&csk->lock);
680
681         if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
682                 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
683                 cxgbi_sock_set_state(csk, CTP_ABORTING);
684                 goto out;
685         }
686
687         cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
688         send_abort_rpl(csk, rst_status);
689
690         if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
691                 csk->err = abort_status_to_errno(csk, req->status, &rst_status);
692                 cxgbi_sock_closed(csk);
693         }
694
695 out:
696         spin_unlock_bh(&csk->lock);
697         cxgbi_sock_put(csk);
698 done:
699         __kfree_skb(skb);
700         return 0;
701 }
702
703 /*
704  * Process ABORT_RPL_RSS CPL message: -> host
705  * Process abort replies.  We only process these messages if we anticipate
706  * them as the coordination between SW and HW in this area is somewhat lacking
707  * and sometimes we get ABORT_RPLs after we are done with the connection that
708  * originated the ABORT_REQ.
709  */
710 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
711 {
712         struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
713         struct cxgbi_sock *csk = ctx;
714
715         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
716                 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
717                 rpl->status, csk, csk ? csk->state : 0,
718                 csk ? csk->flags : 0UL);
719         /*
720          * Ignore replies to post-close aborts indicating that the abort was
721          * requested too late.  These connections are terminated when we get
722          * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
723          * arrives the TID is either no longer used or it has been recycled.
724          */
725         if (rpl->status == CPL_ERR_ABORT_FAILED)
726                 goto rel_skb;
727         /*
728          * Sometimes we've already closed the connection, e.g., a post-close
729          * abort races with ABORT_REQ_RSS, the latter frees the connection
730          * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
731          * but FW turns the ABORT_REQ into a regular one and so we get
732          * ABORT_RPL_RSS with status 0 and no connection.
733          */
734         if (csk)
735                 cxgbi_sock_rcv_abort_rpl(csk);
736 rel_skb:
737         __kfree_skb(skb);
738         return 0;
739 }
740
741 /*
742  * Process RX_ISCSI_HDR CPL message: -> host
743  * Handle received PDUs, the payload could be DDP'ed. If not, the payload
744  * follow after the bhs.
745  */
746 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
747 {
748         struct cxgbi_sock *csk = ctx;
749         struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
750         struct cpl_iscsi_hdr_norss data_cpl;
751         struct cpl_rx_data_ddp_norss ddp_cpl;
752         unsigned int hdr_len, data_len, status;
753         unsigned int len;
754         int err;
755
756         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
757                 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
758                 csk, csk->state, csk->flags, csk->tid, skb, skb->len);
759
760         spin_lock_bh(&csk->lock);
761
762         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
763                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
764                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
765                         csk, csk->state, csk->flags, csk->tid);
766                 if (csk->state != CTP_ABORTING)
767                         goto abort_conn;
768                 else
769                         goto discard;
770         }
771
772         cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
773         cxgbi_skcb_flags(skb) = 0;
774
775         skb_reset_transport_header(skb);
776         __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
777
778         len = hdr_len = ntohs(hdr_cpl->len);
779         /* msg coalesce is off or not enough data received */
780         if (skb->len <= hdr_len) {
781                 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
782                         csk->cdev->ports[csk->port_id]->name, csk->tid,
783                         skb->len, hdr_len);
784                 goto abort_conn;
785         }
786         cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
787
788         err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
789                             sizeof(ddp_cpl));
790         if (err < 0) {
791                 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
792                         csk->cdev->ports[csk->port_id]->name, csk->tid,
793                         skb->len, sizeof(ddp_cpl), err);
794                 goto abort_conn;
795         }
796
797         cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
798         cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
799         cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
800         status = ntohl(ddp_cpl.ddp_status);
801
802         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
803                 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
804                 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
805
806         if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
807                 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
808         if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
809                 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
810         if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
811                 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
812
813         if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
814                 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
815                 if (err < 0) {
816                         pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
817                                 csk->cdev->ports[csk->port_id]->name,
818                                 csk->tid, sizeof(data_cpl), skb->len, err);
819                         goto abort_conn;
820                 }
821                 data_len = ntohs(data_cpl.len);
822                 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
823                         "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
824                         skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
825                 len += sizeof(data_cpl) + data_len;
826         } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
827                 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
828
829         csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
830         __pskb_trim(skb, len);
831         __skb_queue_tail(&csk->receive_queue, skb);
832         cxgbi_conn_pdu_ready(csk);
833
834         spin_unlock_bh(&csk->lock);
835         return 0;
836
837 abort_conn:
838         send_abort_req(csk);
839 discard:
840         spin_unlock_bh(&csk->lock);
841         __kfree_skb(skb);
842         return 0;
843 }
844
845 /*
846  * Process TX_DATA_ACK CPL messages: -> host
847  * Process an acknowledgment of WR completion.  Advance snd_una and send the
848  * next batch of work requests from the write queue.
849  */
850 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
851 {
852         struct cxgbi_sock *csk = ctx;
853         struct cpl_wr_ack *hdr = cplhdr(skb);
854
855         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
856                 "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
857                 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
858
859         cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
860         __kfree_skb(skb);
861         return 0;
862 }
863
864 /*
865  * for each connection, pre-allocate skbs needed for close/abort requests. So
866  * that we can service the request right away.
867  */
868 static int alloc_cpls(struct cxgbi_sock *csk)
869 {
870         csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
871                                         GFP_KERNEL);
872         if (!csk->cpl_close)
873                 return -ENOMEM;
874         csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
875                                         GFP_KERNEL);
876         if (!csk->cpl_abort_req)
877                 goto free_cpl_skbs;
878
879         csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
880                                         GFP_KERNEL);
881         if (!csk->cpl_abort_rpl)
882                 goto free_cpl_skbs;
883
884         return 0;
885
886 free_cpl_skbs:
887         cxgbi_sock_free_cpl_skbs(csk);
888         return -ENOMEM;
889 }
890
891 /**
892  * release_offload_resources - release offload resource
893  * @c3cn: the offloaded iscsi tcp connection.
894  * Release resources held by an offload connection (TID, L2T entry, etc.)
895  */
896 static void l2t_put(struct cxgbi_sock *csk)
897 {
898         struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
899
900         if (csk->l2t) {
901                 l2t_release(L2DATA(t3dev), csk->l2t);
902                 csk->l2t = NULL;
903                 cxgbi_sock_put(csk);
904         }
905 }
906
907 static void release_offload_resources(struct cxgbi_sock *csk)
908 {
909         struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
910
911         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
912                 "csk 0x%p,%u,0x%lx,%u.\n",
913                 csk, csk->state, csk->flags, csk->tid);
914
915         csk->rss_qid = 0;
916         cxgbi_sock_free_cpl_skbs(csk);
917
918         if (csk->wr_cred != csk->wr_max_cred) {
919                 cxgbi_sock_purge_wr_queue(csk);
920                 cxgbi_sock_reset_wr_list(csk);
921         }
922         l2t_put(csk);
923         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
924                 free_atid(csk);
925         else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
926                 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
927                 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
928                 cxgbi_sock_put(csk);
929         }
930         csk->dst = NULL;
931         csk->cdev = NULL;
932 }
933
934 static void update_address(struct cxgbi_hba *chba)
935 {
936         if (chba->ipv4addr) {
937                 if (chba->vdev &&
938                     chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
939                         cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
940                         cxgb3i_set_private_ipv4addr(chba->ndev, 0);
941                         pr_info("%s set %pI4.\n",
942                                 chba->vdev->name, &chba->ipv4addr);
943                 } else if (chba->ipv4addr !=
944                                 cxgb3i_get_private_ipv4addr(chba->ndev)) {
945                         cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
946                         pr_info("%s set %pI4.\n",
947                                 chba->ndev->name, &chba->ipv4addr);
948                 }
949         } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
950                 if (chba->vdev)
951                         cxgb3i_set_private_ipv4addr(chba->vdev, 0);
952                 cxgb3i_set_private_ipv4addr(chba->ndev, 0);
953         }
954 }
955
956 static int init_act_open(struct cxgbi_sock *csk)
957 {
958         struct dst_entry *dst = csk->dst;
959         struct cxgbi_device *cdev = csk->cdev;
960         struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
961         struct net_device *ndev = cdev->ports[csk->port_id];
962         struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
963         struct sk_buff *skb = NULL;
964
965         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
966                 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
967
968         update_address(chba);
969         if (chba->ipv4addr)
970                 csk->saddr.sin_addr.s_addr = chba->ipv4addr;
971
972         csk->rss_qid = 0;
973         csk->l2t = t3_l2t_get(t3dev, dst_get_neighbour(dst), ndev);
974         if (!csk->l2t) {
975                 pr_err("NO l2t available.\n");
976                 return -EINVAL;
977         }
978         cxgbi_sock_get(csk);
979
980         csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
981         if (csk->atid < 0) {
982                 pr_err("NO atid available.\n");
983                 goto rel_resource;
984         }
985         cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
986         cxgbi_sock_get(csk);
987
988         skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
989         if (!skb)
990                 goto rel_resource;
991         skb->sk = (struct sock *)csk;
992         set_arp_failure_handler(skb, act_open_arp_failure);
993
994         csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
995         csk->wr_una_cred = 0;
996         csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
997         cxgbi_sock_reset_wr_list(csk);
998         csk->err = 0;
999
1000         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1001                 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1002                 csk, csk->state, csk->flags,
1003                 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
1004                 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
1005
1006         cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1007         send_act_open_req(csk, skb, csk->l2t);
1008         return 0;
1009
1010 rel_resource:
1011         if (skb)
1012                 __kfree_skb(skb);
1013         return -EINVAL;
1014 }
1015
1016 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
1017         [CPL_ACT_ESTABLISH] = do_act_establish,
1018         [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1019         [CPL_PEER_CLOSE] = do_peer_close,
1020         [CPL_ABORT_REQ_RSS] = do_abort_req,
1021         [CPL_ABORT_RPL_RSS] = do_abort_rpl,
1022         [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1023         [CPL_TX_DMA_ACK] = do_wr_ack,
1024         [CPL_ISCSI_HDR] = do_iscsi_hdr,
1025 };
1026
1027 /**
1028  * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1029  * @cdev:       cxgbi adapter
1030  */
1031 int cxgb3i_ofld_init(struct cxgbi_device *cdev)
1032 {
1033         struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
1034         struct adap_ports port;
1035         struct ofld_page_info rx_page_info;
1036         unsigned int wr_len;
1037         int rc;
1038
1039         if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
1040             t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
1041             t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1042                 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
1043                 return -EINVAL;
1044         }
1045
1046         if (cxgb3i_max_connect > CXGBI_MAX_CONN)
1047                 cxgb3i_max_connect = CXGBI_MAX_CONN;
1048
1049         rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
1050                                         cxgb3i_max_connect);
1051         if (rc < 0)
1052                 return rc;
1053
1054         init_wr_tab(wr_len);
1055         cdev->csk_release_offload_resources = release_offload_resources;
1056         cdev->csk_push_tx_frames = push_tx_frames;
1057         cdev->csk_send_abort_req = send_abort_req;
1058         cdev->csk_send_close_req = send_close_req;
1059         cdev->csk_send_rx_credits = send_rx_credits;
1060         cdev->csk_alloc_cpls = alloc_cpls;
1061         cdev->csk_init_act_open = init_act_open;
1062
1063         pr_info("cdev 0x%p, offload up, added.\n", cdev);
1064         return 0;
1065 }
1066
1067 /*
1068  * functions to program the pagepod in h/w
1069  */
1070 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
1071 {
1072         struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
1073
1074         memset(req, 0, sizeof(*req));
1075
1076         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
1077         req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
1078                                    V_ULPTX_CMD(ULP_MEM_WRITE));
1079         req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
1080                          V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
1081 }
1082
1083 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
1084                         unsigned int idx, unsigned int npods,
1085                                 struct cxgbi_gather_list *gl)
1086 {
1087         struct cxgbi_device *cdev = csk->cdev;
1088         struct cxgbi_ddp_info *ddp = cdev->ddp;
1089         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
1090         int i;
1091
1092         log_debug(1 << CXGBI_DBG_DDP,
1093                 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
1094                 csk, idx, npods, gl);
1095
1096         for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
1097                 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1098                                                 PPOD_SIZE, 0, GFP_ATOMIC);
1099
1100                 if (!skb)
1101                         return -ENOMEM;
1102
1103                 ulp_mem_io_set_hdr(skb, pm_addr);
1104                 cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head +
1105                                         sizeof(struct ulp_mem_io)),
1106                                    hdr, gl, i * PPOD_PAGES_MAX);
1107                 skb->priority = CPL_PRIORITY_CONTROL;
1108                 cxgb3_ofld_send(cdev->lldev, skb);
1109         }
1110         return 0;
1111 }
1112
1113 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
1114                           unsigned int idx, unsigned int npods)
1115 {
1116         struct cxgbi_device *cdev = chba->cdev;
1117         struct cxgbi_ddp_info *ddp = cdev->ddp;
1118         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
1119         int i;
1120
1121         log_debug(1 << CXGBI_DBG_DDP,
1122                 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
1123                 cdev, idx, npods, tag);
1124
1125         for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
1126                 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1127                                                 PPOD_SIZE, 0, GFP_ATOMIC);
1128
1129                 if (!skb) {
1130                         pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
1131                                 tag, idx, i, npods);
1132                         continue;
1133                 }
1134                 ulp_mem_io_set_hdr(skb, pm_addr);
1135                 skb->priority = CPL_PRIORITY_CONTROL;
1136                 cxgb3_ofld_send(cdev->lldev, skb);
1137         }
1138 }
1139
1140 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
1141                                        unsigned int tid, int pg_idx, bool reply)
1142 {
1143         struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1144                                         GFP_KERNEL);
1145         struct cpl_set_tcb_field *req;
1146         u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
1147
1148         log_debug(1 << CXGBI_DBG_DDP,
1149                 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
1150         if (!skb)
1151                 return -ENOMEM;
1152
1153         /* set up ulp submode and page size */
1154         req = (struct cpl_set_tcb_field *)skb->head;
1155         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1156         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1157         req->reply = V_NO_REPLY(reply ? 0 : 1);
1158         req->cpu_idx = 0;
1159         req->word = htons(31);
1160         req->mask = cpu_to_be64(0xF0000000);
1161         req->val = cpu_to_be64(val << 28);
1162         skb->priority = CPL_PRIORITY_CONTROL;
1163
1164         cxgb3_ofld_send(csk->cdev->lldev, skb);
1165         return 0;
1166 }
1167
1168 /**
1169  * cxgb3i_setup_conn_digest - setup conn. digest setting
1170  * @csk: cxgb tcp socket
1171  * @tid: connection id
1172  * @hcrc: header digest enabled
1173  * @dcrc: data digest enabled
1174  * @reply: request reply from h/w
1175  * set up the iscsi digest settings for a connection identified by tid
1176  */
1177 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1178                              int hcrc, int dcrc, int reply)
1179 {
1180         struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1181                                         GFP_KERNEL);
1182         struct cpl_set_tcb_field *req;
1183         u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
1184
1185         log_debug(1 << CXGBI_DBG_DDP,
1186                 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
1187         if (!skb)
1188                 return -ENOMEM;
1189
1190         /* set up ulp submode and page size */
1191         req = (struct cpl_set_tcb_field *)skb->head;
1192         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1193         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1194         req->reply = V_NO_REPLY(reply ? 0 : 1);
1195         req->cpu_idx = 0;
1196         req->word = htons(31);
1197         req->mask = cpu_to_be64(0x0F000000);
1198         req->val = cpu_to_be64(val << 24);
1199         skb->priority = CPL_PRIORITY_CONTROL;
1200
1201         cxgb3_ofld_send(csk->cdev->lldev, skb);
1202         return 0;
1203 }
1204
1205 /**
1206  * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource
1207  * @cdev: cxgb3i adapter
1208  * release all the resource held by the ddp pagepod manager for a given
1209  * adapter if needed
1210  */
1211
1212 static void t3_ddp_cleanup(struct cxgbi_device *cdev)
1213 {
1214         struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1215
1216         if (cxgbi_ddp_cleanup(cdev)) {
1217                 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev);
1218                 tdev->ulp_iscsi = NULL;
1219         }
1220 }
1221
1222 /**
1223  * ddp_init - initialize the cxgb3 adapter's ddp resource
1224  * @cdev: cxgb3i adapter
1225  * initialize the ddp pagepod manager for a given adapter
1226  */
1227 static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
1228 {
1229         struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1230         struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi;
1231         struct ulp_iscsi_info uinfo;
1232         unsigned int pgsz_factor[4];
1233         int i, err;
1234
1235         if (ddp) {
1236                 kref_get(&ddp->refcnt);
1237                 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
1238                         tdev, tdev->ulp_iscsi);
1239                 cdev->ddp = ddp;
1240                 return -EALREADY;
1241         }
1242
1243         err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1244         if (err < 0) {
1245                 pr_err("%s, failed to get iscsi param err=%d.\n",
1246                          tdev->name, err);
1247                 return err;
1248         }
1249
1250         err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit,
1251                         uinfo.max_txsz, uinfo.max_rxsz);
1252         if (err < 0)
1253                 return err;
1254
1255         ddp = cdev->ddp;
1256
1257         uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
1258         cxgbi_ddp_page_size_factor(pgsz_factor);
1259         for (i = 0; i < 4; i++)
1260                 uinfo.pgsz_factor[i] = pgsz_factor[i];
1261         uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT);
1262
1263         err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1264         if (err < 0) {
1265                 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
1266                         tdev->name, err);
1267                 cxgbi_ddp_cleanup(cdev);
1268                 return err;
1269         }
1270         tdev->ulp_iscsi = ddp;
1271
1272         cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1273         cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1274         cdev->csk_ddp_set = ddp_set_map;
1275         cdev->csk_ddp_clear = ddp_clear_map;
1276
1277         pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1278                 "%u/%u.\n",
1279                 tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
1280                 ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz,
1281                 ddp->max_rxsz, uinfo.max_rxsz);
1282         return 0;
1283 }
1284
1285 static void cxgb3i_dev_close(struct t3cdev *t3dev)
1286 {
1287         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1288
1289         if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
1290                 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
1291                 return;
1292         }
1293
1294         cxgbi_device_unregister(cdev);
1295 }
1296
1297 /**
1298  * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1299  * @t3dev: t3cdev adapter
1300  */
1301 static void cxgb3i_dev_open(struct t3cdev *t3dev)
1302 {
1303         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1304         struct adapter *adapter = tdev2adap(t3dev);
1305         int i, err;
1306
1307         if (cdev) {
1308                 pr_info("0x%p, updating.\n", cdev);
1309                 return;
1310         }
1311
1312         cdev = cxgbi_device_register(0, adapter->params.nports);
1313         if (!cdev) {
1314                 pr_warn("device 0x%p register failed.\n", t3dev);
1315                 return;
1316         }
1317
1318         cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
1319         cdev->lldev = t3dev;
1320         cdev->pdev = adapter->pdev;
1321         cdev->ports = adapter->port;
1322         cdev->nports = adapter->params.nports;
1323         cdev->mtus = adapter->params.mtus;
1324         cdev->nmtus = NMTUS;
1325         cdev->snd_win = cxgb3i_snd_win;
1326         cdev->rcv_win = cxgb3i_rcv_win;
1327         cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
1328         cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
1329         cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
1330         cdev->dev_ddp_cleanup = t3_ddp_cleanup;
1331         cdev->itp = &cxgb3i_iscsi_transport;
1332
1333         err = cxgb3i_ddp_init(cdev);
1334         if (err) {
1335                 pr_info("0x%p ddp init failed\n", cdev);
1336                 goto err_out;
1337         }
1338
1339         err = cxgb3i_ofld_init(cdev);
1340         if (err) {
1341                 pr_info("0x%p offload init failed\n", cdev);
1342                 goto err_out;
1343         }
1344
1345         err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
1346                                 &cxgb3i_host_template, cxgb3i_stt);
1347         if (err)
1348                 goto err_out;
1349
1350         for (i = 0; i < cdev->nports; i++)
1351                 cdev->hbas[i]->ipv4addr =
1352                         cxgb3i_get_private_ipv4addr(cdev->ports[i]);
1353
1354         pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1355                 cdev, cdev ? cdev->flags : 0, t3dev, err);
1356         return;
1357
1358 err_out:
1359         cxgbi_device_unregister(cdev);
1360 }
1361
1362 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
1363 {
1364         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1365
1366         log_debug(1 << CXGBI_DBG_TOE,
1367                 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1368                 t3dev, cdev, event, port);
1369         if (!cdev)
1370                 return;
1371
1372         switch (event) {
1373         case OFFLOAD_STATUS_DOWN:
1374                 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
1375                 break;
1376         case OFFLOAD_STATUS_UP:
1377                 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
1378                 break;
1379         }
1380 }
1381
1382 /**
1383  * cxgb3i_init_module - module init entry point
1384  *
1385  * initialize any driver wide global data structures and register itself
1386  *      with the cxgb3 module
1387  */
1388 static int __init cxgb3i_init_module(void)
1389 {
1390         int rc;
1391
1392         printk(KERN_INFO "%s", version);
1393
1394         rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1395         if (rc < 0)
1396                 return rc;
1397
1398         cxgb3_register_client(&t3_client);
1399         return 0;
1400 }
1401
1402 /**
1403  * cxgb3i_exit_module - module cleanup/exit entry point
1404  *
1405  * go through the driver hba list and for each hba, release any resource held.
1406  *      and unregisters iscsi transport and the cxgb3 module
1407  */
1408 static void __exit cxgb3i_exit_module(void)
1409 {
1410         cxgb3_unregister_client(&t3_client);
1411         cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
1412         cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1413 }
1414
1415 module_init(cxgb3i_init_module);
1416 module_exit(cxgb3i_exit_module);