93a3c708411a813a7afa1614af752659cad8f6c3
[linux-2.6.git] / drivers / scsi / be2iscsi / be_main.c
1 /**
2  * Copyright (C) 2005 - 2011 Emulex
3  * All rights reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License version 2
7  * as published by the Free Software Foundation.  The full GNU General
8  * Public License is included in this distribution in the file called COPYING.
9  *
10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
11  *
12  * Contact Information:
13  * linux-drivers@emulex.com
14  *
15  * Emulex
16  * 3333 Susan Street
17  * Costa Mesa, CA 92626
18  */
19
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/string.h>
27 #include <linux/kernel.h>
28 #include <linux/semaphore.h>
29 #include <linux/iscsi_boot_sysfs.h>
30
31 #include <scsi/libiscsi.h>
32 #include <scsi/scsi_transport_iscsi.h>
33 #include <scsi/scsi_transport.h>
34 #include <scsi/scsi_cmnd.h>
35 #include <scsi/scsi_device.h>
36 #include <scsi/scsi_host.h>
37 #include <scsi/scsi.h>
38 #include "be_main.h"
39 #include "be_iscsi.h"
40 #include "be_mgmt.h"
41
42 static unsigned int be_iopoll_budget = 10;
43 static unsigned int be_max_phys_size = 64;
44 static unsigned int enable_msix = 1;
45 static unsigned int gcrashmode = 0;
46 static unsigned int num_hba = 0;
47
48 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
49 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
50 MODULE_AUTHOR("ServerEngines Corporation");
51 MODULE_LICENSE("GPL");
52 module_param(be_iopoll_budget, int, 0);
53 module_param(enable_msix, int, 0);
54 module_param(be_max_phys_size, uint, S_IRUGO);
55 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
56                                    "contiguous memory that can be allocated."
57                                    "Range is 16 - 128");
58
59 static int beiscsi_slave_configure(struct scsi_device *sdev)
60 {
61         blk_queue_max_segment_size(sdev->request_queue, 65536);
62         return 0;
63 }
64
65 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
66 {
67         struct iscsi_cls_session *cls_session;
68         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
69         struct beiscsi_io_task *aborted_io_task;
70         struct iscsi_conn *conn;
71         struct beiscsi_conn *beiscsi_conn;
72         struct beiscsi_hba *phba;
73         struct iscsi_session *session;
74         struct invalidate_command_table *inv_tbl;
75         struct be_dma_mem nonemb_cmd;
76         unsigned int cid, tag, num_invalidate;
77
78         cls_session = starget_to_session(scsi_target(sc->device));
79         session = cls_session->dd_data;
80
81         spin_lock_bh(&session->lock);
82         if (!aborted_task || !aborted_task->sc) {
83                 /* we raced */
84                 spin_unlock_bh(&session->lock);
85                 return SUCCESS;
86         }
87
88         aborted_io_task = aborted_task->dd_data;
89         if (!aborted_io_task->scsi_cmnd) {
90                 /* raced or invalid command */
91                 spin_unlock_bh(&session->lock);
92                 return SUCCESS;
93         }
94         spin_unlock_bh(&session->lock);
95         conn = aborted_task->conn;
96         beiscsi_conn = conn->dd_data;
97         phba = beiscsi_conn->phba;
98
99         /* invalidate iocb */
100         cid = beiscsi_conn->beiscsi_conn_cid;
101         inv_tbl = phba->inv_tbl;
102         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
103         inv_tbl->cid = cid;
104         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
105         num_invalidate = 1;
106         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
107                                 sizeof(struct invalidate_commands_params_in),
108                                 &nonemb_cmd.dma);
109         if (nonemb_cmd.va == NULL) {
110                 SE_DEBUG(DBG_LVL_1,
111                          "Failed to allocate memory for"
112                          "mgmt_invalidate_icds\n");
113                 return FAILED;
114         }
115         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
116
117         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
118                                    cid, &nonemb_cmd);
119         if (!tag) {
120                 shost_printk(KERN_WARNING, phba->shost,
121                              "mgmt_invalidate_icds could not be"
122                              " submitted\n");
123                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
124                                     nonemb_cmd.va, nonemb_cmd.dma);
125
126                 return FAILED;
127         } else {
128                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
129                                          phba->ctrl.mcc_numtag[tag]);
130                 free_mcc_tag(&phba->ctrl, tag);
131         }
132         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
133                             nonemb_cmd.va, nonemb_cmd.dma);
134         return iscsi_eh_abort(sc);
135 }
136
137 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
138 {
139         struct iscsi_task *abrt_task;
140         struct beiscsi_io_task *abrt_io_task;
141         struct iscsi_conn *conn;
142         struct beiscsi_conn *beiscsi_conn;
143         struct beiscsi_hba *phba;
144         struct iscsi_session *session;
145         struct iscsi_cls_session *cls_session;
146         struct invalidate_command_table *inv_tbl;
147         struct be_dma_mem nonemb_cmd;
148         unsigned int cid, tag, i, num_invalidate;
149         int rc = FAILED;
150
151         /* invalidate iocbs */
152         cls_session = starget_to_session(scsi_target(sc->device));
153         session = cls_session->dd_data;
154         spin_lock_bh(&session->lock);
155         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
156                 goto unlock;
157
158         conn = session->leadconn;
159         beiscsi_conn = conn->dd_data;
160         phba = beiscsi_conn->phba;
161         cid = beiscsi_conn->beiscsi_conn_cid;
162         inv_tbl = phba->inv_tbl;
163         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
164         num_invalidate = 0;
165         for (i = 0; i < conn->session->cmds_max; i++) {
166                 abrt_task = conn->session->cmds[i];
167                 abrt_io_task = abrt_task->dd_data;
168                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
169                         continue;
170
171                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
172                         continue;
173
174                 inv_tbl->cid = cid;
175                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
176                 num_invalidate++;
177                 inv_tbl++;
178         }
179         spin_unlock_bh(&session->lock);
180         inv_tbl = phba->inv_tbl;
181
182         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
183                                 sizeof(struct invalidate_commands_params_in),
184                                 &nonemb_cmd.dma);
185         if (nonemb_cmd.va == NULL) {
186                 SE_DEBUG(DBG_LVL_1,
187                          "Failed to allocate memory for"
188                          "mgmt_invalidate_icds\n");
189                 return FAILED;
190         }
191         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
192         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
193         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
194                                    cid, &nonemb_cmd);
195         if (!tag) {
196                 shost_printk(KERN_WARNING, phba->shost,
197                              "mgmt_invalidate_icds could not be"
198                              " submitted\n");
199                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
200                                     nonemb_cmd.va, nonemb_cmd.dma);
201                 return FAILED;
202         } else {
203                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
204                                          phba->ctrl.mcc_numtag[tag]);
205                 free_mcc_tag(&phba->ctrl, tag);
206         }
207         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
208                             nonemb_cmd.va, nonemb_cmd.dma);
209         return iscsi_eh_device_reset(sc);
210 unlock:
211         spin_unlock_bh(&session->lock);
212         return rc;
213 }
214
215 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
216 {
217         struct beiscsi_hba *phba = data;
218         struct mgmt_session_info *boot_sess = &phba->boot_sess;
219         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
220         char *str = buf;
221         int rc;
222
223         switch (type) {
224         case ISCSI_BOOT_TGT_NAME:
225                 rc = sprintf(buf, "%.*s\n",
226                             (int)strlen(boot_sess->target_name),
227                             (char *)&boot_sess->target_name);
228                 break;
229         case ISCSI_BOOT_TGT_IP_ADDR:
230                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
231                         rc = sprintf(buf, "%pI4\n",
232                                 (char *)&boot_conn->dest_ipaddr.ip_address);
233                 else
234                         rc = sprintf(str, "%pI6\n",
235                                 (char *)&boot_conn->dest_ipaddr.ip_address);
236                 break;
237         case ISCSI_BOOT_TGT_PORT:
238                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
239                 break;
240
241         case ISCSI_BOOT_TGT_CHAP_NAME:
242                 rc = sprintf(str,  "%.*s\n",
243                              boot_conn->negotiated_login_options.auth_data.chap.
244                              target_chap_name_length,
245                              (char *)&boot_conn->negotiated_login_options.
246                              auth_data.chap.target_chap_name);
247                 break;
248         case ISCSI_BOOT_TGT_CHAP_SECRET:
249                 rc = sprintf(str,  "%.*s\n",
250                              boot_conn->negotiated_login_options.auth_data.chap.
251                              target_secret_length,
252                              (char *)&boot_conn->negotiated_login_options.
253                              auth_data.chap.target_secret);
254                 break;
255         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
256                 rc = sprintf(str,  "%.*s\n",
257                              boot_conn->negotiated_login_options.auth_data.chap.
258                              intr_chap_name_length,
259                              (char *)&boot_conn->negotiated_login_options.
260                              auth_data.chap.intr_chap_name);
261                 break;
262         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
263                 rc = sprintf(str,  "%.*s\n",
264                              boot_conn->negotiated_login_options.auth_data.chap.
265                              intr_secret_length,
266                              (char *)&boot_conn->negotiated_login_options.
267                              auth_data.chap.intr_secret);
268                 break;
269         case ISCSI_BOOT_TGT_FLAGS:
270                 rc = sprintf(str, "2\n");
271                 break;
272         case ISCSI_BOOT_TGT_NIC_ASSOC:
273                 rc = sprintf(str, "0\n");
274                 break;
275         default:
276                 rc = -ENOSYS;
277                 break;
278         }
279         return rc;
280 }
281
282 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
283 {
284         struct beiscsi_hba *phba = data;
285         char *str = buf;
286         int rc;
287
288         switch (type) {
289         case ISCSI_BOOT_INI_INITIATOR_NAME:
290                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
291                 break;
292         default:
293                 rc = -ENOSYS;
294                 break;
295         }
296         return rc;
297 }
298
299 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
300 {
301         struct beiscsi_hba *phba = data;
302         char *str = buf;
303         int rc;
304
305         switch (type) {
306         case ISCSI_BOOT_ETH_FLAGS:
307                 rc = sprintf(str, "2\n");
308                 break;
309         case ISCSI_BOOT_ETH_INDEX:
310                 rc = sprintf(str, "0\n");
311                 break;
312         case ISCSI_BOOT_ETH_MAC:
313                 rc  = beiscsi_get_macaddr(buf, phba);
314                 if (rc < 0) {
315                         SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
316                         return rc;
317                 }
318         break;
319         default:
320                 rc = -ENOSYS;
321                 break;
322         }
323         return rc;
324 }
325
326
327 static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
328 {
329         int rc;
330
331         switch (type) {
332         case ISCSI_BOOT_TGT_NAME:
333         case ISCSI_BOOT_TGT_IP_ADDR:
334         case ISCSI_BOOT_TGT_PORT:
335         case ISCSI_BOOT_TGT_CHAP_NAME:
336         case ISCSI_BOOT_TGT_CHAP_SECRET:
337         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
338         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
339         case ISCSI_BOOT_TGT_NIC_ASSOC:
340         case ISCSI_BOOT_TGT_FLAGS:
341                 rc = S_IRUGO;
342                 break;
343         default:
344                 rc = 0;
345                 break;
346         }
347         return rc;
348 }
349
350 static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
351 {
352         int rc;
353
354         switch (type) {
355         case ISCSI_BOOT_INI_INITIATOR_NAME:
356                 rc = S_IRUGO;
357                 break;
358         default:
359                 rc = 0;
360                 break;
361         }
362         return rc;
363 }
364
365
366 static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
367 {
368         int rc;
369
370         switch (type) {
371         case ISCSI_BOOT_ETH_FLAGS:
372         case ISCSI_BOOT_ETH_MAC:
373         case ISCSI_BOOT_ETH_INDEX:
374                 rc = S_IRUGO;
375                 break;
376         default:
377                 rc = 0;
378                 break;
379         }
380         return rc;
381 }
382
383 /*------------------- PCI Driver operations and data ----------------- */
384 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
385         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
386         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
387         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
388         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
389         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
390         { 0 }
391 };
392 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
393
394 static struct scsi_host_template beiscsi_sht = {
395         .module = THIS_MODULE,
396         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
397         .proc_name = DRV_NAME,
398         .queuecommand = iscsi_queuecommand,
399         .change_queue_depth = iscsi_change_queue_depth,
400         .slave_configure = beiscsi_slave_configure,
401         .target_alloc = iscsi_target_alloc,
402         .eh_abort_handler = beiscsi_eh_abort,
403         .eh_device_reset_handler = beiscsi_eh_device_reset,
404         .eh_target_reset_handler = iscsi_eh_session_reset,
405         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
406         .can_queue = BE2_IO_DEPTH,
407         .this_id = -1,
408         .max_sectors = BEISCSI_MAX_SECTORS,
409         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
410         .use_clustering = ENABLE_CLUSTERING,
411 };
412
413 static struct scsi_transport_template *beiscsi_scsi_transport;
414
415 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
416 {
417         struct beiscsi_hba *phba;
418         struct Scsi_Host *shost;
419
420         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
421         if (!shost) {
422                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
423                         "iscsi_host_alloc failed\n");
424                 return NULL;
425         }
426         shost->dma_boundary = pcidev->dma_mask;
427         shost->max_id = BE2_MAX_SESSIONS;
428         shost->max_channel = 0;
429         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
430         shost->max_lun = BEISCSI_NUM_MAX_LUN;
431         shost->transportt = beiscsi_scsi_transport;
432         phba = iscsi_host_priv(shost);
433         memset(phba, 0, sizeof(*phba));
434         phba->shost = shost;
435         phba->pcidev = pci_dev_get(pcidev);
436         pci_set_drvdata(pcidev, phba);
437
438         if (iscsi_host_add(shost, &phba->pcidev->dev))
439                 goto free_devices;
440
441         return phba;
442
443 free_devices:
444         pci_dev_put(phba->pcidev);
445         iscsi_host_free(phba->shost);
446         return NULL;
447 }
448
449 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
450 {
451         if (phba->csr_va) {
452                 iounmap(phba->csr_va);
453                 phba->csr_va = NULL;
454         }
455         if (phba->db_va) {
456                 iounmap(phba->db_va);
457                 phba->db_va = NULL;
458         }
459         if (phba->pci_va) {
460                 iounmap(phba->pci_va);
461                 phba->pci_va = NULL;
462         }
463 }
464
465 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
466                                 struct pci_dev *pcidev)
467 {
468         u8 __iomem *addr;
469         int pcicfg_reg;
470
471         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
472                                pci_resource_len(pcidev, 2));
473         if (addr == NULL)
474                 return -ENOMEM;
475         phba->ctrl.csr = addr;
476         phba->csr_va = addr;
477         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
478
479         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
480         if (addr == NULL)
481                 goto pci_map_err;
482         phba->ctrl.db = addr;
483         phba->db_va = addr;
484         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
485
486         if (phba->generation == BE_GEN2)
487                 pcicfg_reg = 1;
488         else
489                 pcicfg_reg = 0;
490
491         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
492                                pci_resource_len(pcidev, pcicfg_reg));
493
494         if (addr == NULL)
495                 goto pci_map_err;
496         phba->ctrl.pcicfg = addr;
497         phba->pci_va = addr;
498         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
499         return 0;
500
501 pci_map_err:
502         beiscsi_unmap_pci_function(phba);
503         return -ENOMEM;
504 }
505
506 static int beiscsi_enable_pci(struct pci_dev *pcidev)
507 {
508         int ret;
509
510         ret = pci_enable_device(pcidev);
511         if (ret) {
512                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
513                         "failed. Returning -ENODEV\n");
514                 return ret;
515         }
516
517         pci_set_master(pcidev);
518         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
519                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
520                 if (ret) {
521                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
522                         pci_disable_device(pcidev);
523                         return ret;
524                 }
525         }
526         return 0;
527 }
528
529 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
530 {
531         struct be_ctrl_info *ctrl = &phba->ctrl;
532         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
533         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
534         int status = 0;
535
536         ctrl->pdev = pdev;
537         status = beiscsi_map_pci_bars(phba, pdev);
538         if (status)
539                 return status;
540         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
541         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
542                                                   mbox_mem_alloc->size,
543                                                   &mbox_mem_alloc->dma);
544         if (!mbox_mem_alloc->va) {
545                 beiscsi_unmap_pci_function(phba);
546                 status = -ENOMEM;
547                 return status;
548         }
549
550         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
551         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
552         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
553         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
554         spin_lock_init(&ctrl->mbox_lock);
555         spin_lock_init(&phba->ctrl.mcc_lock);
556         spin_lock_init(&phba->ctrl.mcc_cq_lock);
557
558         return status;
559 }
560
561 static void beiscsi_get_params(struct beiscsi_hba *phba)
562 {
563         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
564                                     - (phba->fw_config.iscsi_cid_count
565                                     + BE2_TMFS
566                                     + BE2_NOPOUT_REQ));
567         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
568         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
569         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
570         phba->params.num_sge_per_io = BE2_SGE;
571         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
572         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
573         phba->params.eq_timer = 64;
574         phba->params.num_eq_entries =
575             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
576                                     + BE2_TMFS) / 512) + 1) * 512;
577         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
578                                 ? 1024 : phba->params.num_eq_entries;
579         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
580                              phba->params.num_eq_entries);
581         phba->params.num_cq_entries =
582             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
583                                     + BE2_TMFS) / 512) + 1) * 512;
584         phba->params.wrbs_per_cxn = 256;
585 }
586
587 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
588                            unsigned int id, unsigned int clr_interrupt,
589                            unsigned int num_processed,
590                            unsigned char rearm, unsigned char event)
591 {
592         u32 val = 0;
593         val |= id & DB_EQ_RING_ID_MASK;
594         if (rearm)
595                 val |= 1 << DB_EQ_REARM_SHIFT;
596         if (clr_interrupt)
597                 val |= 1 << DB_EQ_CLR_SHIFT;
598         if (event)
599                 val |= 1 << DB_EQ_EVNT_SHIFT;
600         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
601         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
602 }
603
604 /**
605  * be_isr_mcc - The isr routine of the driver.
606  * @irq: Not used
607  * @dev_id: Pointer to host adapter structure
608  */
609 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
610 {
611         struct beiscsi_hba *phba;
612         struct be_eq_entry *eqe = NULL;
613         struct be_queue_info *eq;
614         struct be_queue_info *mcc;
615         unsigned int num_eq_processed;
616         struct be_eq_obj *pbe_eq;
617         unsigned long flags;
618
619         pbe_eq = dev_id;
620         eq = &pbe_eq->q;
621         phba =  pbe_eq->phba;
622         mcc = &phba->ctrl.mcc_obj.cq;
623         eqe = queue_tail_node(eq);
624         if (!eqe)
625                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
626
627         num_eq_processed = 0;
628
629         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
630                                 & EQE_VALID_MASK) {
631                 if (((eqe->dw[offsetof(struct amap_eq_entry,
632                      resource_id) / 32] &
633                      EQE_RESID_MASK) >> 16) == mcc->id) {
634                         spin_lock_irqsave(&phba->isr_lock, flags);
635                         phba->todo_mcc_cq = 1;
636                         spin_unlock_irqrestore(&phba->isr_lock, flags);
637                 }
638                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
639                 queue_tail_inc(eq);
640                 eqe = queue_tail_node(eq);
641                 num_eq_processed++;
642         }
643         if (phba->todo_mcc_cq)
644                 queue_work(phba->wq, &phba->work_cqs);
645         if (num_eq_processed)
646                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
647
648         return IRQ_HANDLED;
649 }
650
651 /**
652  * be_isr_msix - The isr routine of the driver.
653  * @irq: Not used
654  * @dev_id: Pointer to host adapter structure
655  */
656 static irqreturn_t be_isr_msix(int irq, void *dev_id)
657 {
658         struct beiscsi_hba *phba;
659         struct be_eq_entry *eqe = NULL;
660         struct be_queue_info *eq;
661         struct be_queue_info *cq;
662         unsigned int num_eq_processed;
663         struct be_eq_obj *pbe_eq;
664         unsigned long flags;
665
666         pbe_eq = dev_id;
667         eq = &pbe_eq->q;
668         cq = pbe_eq->cq;
669         eqe = queue_tail_node(eq);
670         if (!eqe)
671                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
672
673         phba = pbe_eq->phba;
674         num_eq_processed = 0;
675         if (blk_iopoll_enabled) {
676                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
677                                         & EQE_VALID_MASK) {
678                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
679                                 blk_iopoll_sched(&pbe_eq->iopoll);
680
681                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
682                         queue_tail_inc(eq);
683                         eqe = queue_tail_node(eq);
684                         num_eq_processed++;
685                 }
686                 if (num_eq_processed)
687                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
688
689                 return IRQ_HANDLED;
690         } else {
691                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
692                                                 & EQE_VALID_MASK) {
693                         spin_lock_irqsave(&phba->isr_lock, flags);
694                         phba->todo_cq = 1;
695                         spin_unlock_irqrestore(&phba->isr_lock, flags);
696                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
697                         queue_tail_inc(eq);
698                         eqe = queue_tail_node(eq);
699                         num_eq_processed++;
700                 }
701                 if (phba->todo_cq)
702                         queue_work(phba->wq, &phba->work_cqs);
703
704                 if (num_eq_processed)
705                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
706
707                 return IRQ_HANDLED;
708         }
709 }
710
711 /**
712  * be_isr - The isr routine of the driver.
713  * @irq: Not used
714  * @dev_id: Pointer to host adapter structure
715  */
716 static irqreturn_t be_isr(int irq, void *dev_id)
717 {
718         struct beiscsi_hba *phba;
719         struct hwi_controller *phwi_ctrlr;
720         struct hwi_context_memory *phwi_context;
721         struct be_eq_entry *eqe = NULL;
722         struct be_queue_info *eq;
723         struct be_queue_info *cq;
724         struct be_queue_info *mcc;
725         unsigned long flags, index;
726         unsigned int num_mcceq_processed, num_ioeq_processed;
727         struct be_ctrl_info *ctrl;
728         struct be_eq_obj *pbe_eq;
729         int isr;
730
731         phba = dev_id;
732         ctrl = &phba->ctrl;
733         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
734                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
735         if (!isr)
736                 return IRQ_NONE;
737
738         phwi_ctrlr = phba->phwi_ctrlr;
739         phwi_context = phwi_ctrlr->phwi_ctxt;
740         pbe_eq = &phwi_context->be_eq[0];
741
742         eq = &phwi_context->be_eq[0].q;
743         mcc = &phba->ctrl.mcc_obj.cq;
744         index = 0;
745         eqe = queue_tail_node(eq);
746         if (!eqe)
747                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
748
749         num_ioeq_processed = 0;
750         num_mcceq_processed = 0;
751         if (blk_iopoll_enabled) {
752                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
753                                         & EQE_VALID_MASK) {
754                         if (((eqe->dw[offsetof(struct amap_eq_entry,
755                              resource_id) / 32] &
756                              EQE_RESID_MASK) >> 16) == mcc->id) {
757                                 spin_lock_irqsave(&phba->isr_lock, flags);
758                                 phba->todo_mcc_cq = 1;
759                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
760                                 num_mcceq_processed++;
761                         } else {
762                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
763                                         blk_iopoll_sched(&pbe_eq->iopoll);
764                                 num_ioeq_processed++;
765                         }
766                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
767                         queue_tail_inc(eq);
768                         eqe = queue_tail_node(eq);
769                 }
770                 if (num_ioeq_processed || num_mcceq_processed) {
771                         if (phba->todo_mcc_cq)
772                                 queue_work(phba->wq, &phba->work_cqs);
773
774                         if ((num_mcceq_processed) && (!num_ioeq_processed))
775                                 hwi_ring_eq_db(phba, eq->id, 0,
776                                               (num_ioeq_processed +
777                                                num_mcceq_processed) , 1, 1);
778                         else
779                                 hwi_ring_eq_db(phba, eq->id, 0,
780                                                (num_ioeq_processed +
781                                                 num_mcceq_processed), 0, 1);
782
783                         return IRQ_HANDLED;
784                 } else
785                         return IRQ_NONE;
786         } else {
787                 cq = &phwi_context->be_cq[0];
788                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
789                                                 & EQE_VALID_MASK) {
790
791                         if (((eqe->dw[offsetof(struct amap_eq_entry,
792                              resource_id) / 32] &
793                              EQE_RESID_MASK) >> 16) != cq->id) {
794                                 spin_lock_irqsave(&phba->isr_lock, flags);
795                                 phba->todo_mcc_cq = 1;
796                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
797                         } else {
798                                 spin_lock_irqsave(&phba->isr_lock, flags);
799                                 phba->todo_cq = 1;
800                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
801                         }
802                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
803                         queue_tail_inc(eq);
804                         eqe = queue_tail_node(eq);
805                         num_ioeq_processed++;
806                 }
807                 if (phba->todo_cq || phba->todo_mcc_cq)
808                         queue_work(phba->wq, &phba->work_cqs);
809
810                 if (num_ioeq_processed) {
811                         hwi_ring_eq_db(phba, eq->id, 0,
812                                        num_ioeq_processed, 1, 1);
813                         return IRQ_HANDLED;
814                 } else
815                         return IRQ_NONE;
816         }
817 }
818
819 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
820 {
821         struct pci_dev *pcidev = phba->pcidev;
822         struct hwi_controller *phwi_ctrlr;
823         struct hwi_context_memory *phwi_context;
824         int ret, msix_vec, i, j;
825         char desc[32];
826
827         phwi_ctrlr = phba->phwi_ctrlr;
828         phwi_context = phwi_ctrlr->phwi_ctxt;
829
830         if (phba->msix_enabled) {
831                 for (i = 0; i < phba->num_cpus; i++) {
832                         sprintf(desc, "beiscsi_msix_%04x", i);
833                         msix_vec = phba->msix_entries[i].vector;
834                         ret = request_irq(msix_vec, be_isr_msix, 0, desc,
835                                           &phwi_context->be_eq[i]);
836                         if (ret) {
837                                 shost_printk(KERN_ERR, phba->shost,
838                                              "beiscsi_init_irqs-Failed to"
839                                              "register msix for i = %d\n", i);
840                                 if (!i)
841                                         return ret;
842                                 goto free_msix_irqs;
843                         }
844                 }
845                 msix_vec = phba->msix_entries[i].vector;
846                 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
847                                   &phwi_context->be_eq[i]);
848                 if (ret) {
849                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
850                                      "Failed to register beiscsi_msix_mcc\n");
851                         i++;
852                         goto free_msix_irqs;
853                 }
854
855         } else {
856                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
857                                   "beiscsi", phba);
858                 if (ret) {
859                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
860                                      "Failed to register irq\\n");
861                         return ret;
862                 }
863         }
864         return 0;
865 free_msix_irqs:
866         for (j = i - 1; j == 0; j++)
867                 free_irq(msix_vec, &phwi_context->be_eq[j]);
868         return ret;
869 }
870
871 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
872                            unsigned int id, unsigned int num_processed,
873                            unsigned char rearm, unsigned char event)
874 {
875         u32 val = 0;
876         val |= id & DB_CQ_RING_ID_MASK;
877         if (rearm)
878                 val |= 1 << DB_CQ_REARM_SHIFT;
879         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
880         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
881 }
882
883 static unsigned int
884 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
885                           struct beiscsi_hba *phba,
886                           unsigned short cid,
887                           struct pdu_base *ppdu,
888                           unsigned long pdu_len,
889                           void *pbuffer, unsigned long buf_len)
890 {
891         struct iscsi_conn *conn = beiscsi_conn->conn;
892         struct iscsi_session *session = conn->session;
893         struct iscsi_task *task;
894         struct beiscsi_io_task *io_task;
895         struct iscsi_hdr *login_hdr;
896
897         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
898                                                 PDUBASE_OPCODE_MASK) {
899         case ISCSI_OP_NOOP_IN:
900                 pbuffer = NULL;
901                 buf_len = 0;
902                 break;
903         case ISCSI_OP_ASYNC_EVENT:
904                 break;
905         case ISCSI_OP_REJECT:
906                 WARN_ON(!pbuffer);
907                 WARN_ON(!(buf_len == 48));
908                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
909                 break;
910         case ISCSI_OP_LOGIN_RSP:
911         case ISCSI_OP_TEXT_RSP:
912                 task = conn->login_task;
913                 io_task = task->dd_data;
914                 login_hdr = (struct iscsi_hdr *)ppdu;
915                 login_hdr->itt = io_task->libiscsi_itt;
916                 break;
917         default:
918                 shost_printk(KERN_WARNING, phba->shost,
919                              "Unrecognized opcode 0x%x in async msg\n",
920                              (ppdu->
921                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
922                                                 & PDUBASE_OPCODE_MASK));
923                 return 1;
924         }
925
926         spin_lock_bh(&session->lock);
927         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
928         spin_unlock_bh(&session->lock);
929         return 0;
930 }
931
932 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
933 {
934         struct sgl_handle *psgl_handle;
935
936         if (phba->io_sgl_hndl_avbl) {
937                 SE_DEBUG(DBG_LVL_8,
938                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
939                          phba->io_sgl_alloc_index);
940                 psgl_handle = phba->io_sgl_hndl_base[phba->
941                                                 io_sgl_alloc_index];
942                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
943                 phba->io_sgl_hndl_avbl--;
944                 if (phba->io_sgl_alloc_index == (phba->params.
945                                                  ios_per_ctrl - 1))
946                         phba->io_sgl_alloc_index = 0;
947                 else
948                         phba->io_sgl_alloc_index++;
949         } else
950                 psgl_handle = NULL;
951         return psgl_handle;
952 }
953
954 static void
955 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
956 {
957         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
958                  phba->io_sgl_free_index);
959         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
960                 /*
961                  * this can happen if clean_task is called on a task that
962                  * failed in xmit_task or alloc_pdu.
963                  */
964                  SE_DEBUG(DBG_LVL_8,
965                          "Double Free in IO SGL io_sgl_free_index=%d,"
966                          "value there=%p\n", phba->io_sgl_free_index,
967                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
968                 return;
969         }
970         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
971         phba->io_sgl_hndl_avbl++;
972         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
973                 phba->io_sgl_free_index = 0;
974         else
975                 phba->io_sgl_free_index++;
976 }
977
978 /**
979  * alloc_wrb_handle - To allocate a wrb handle
980  * @phba: The hba pointer
981  * @cid: The cid to use for allocation
982  *
983  * This happens under session_lock until submission to chip
984  */
985 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
986 {
987         struct hwi_wrb_context *pwrb_context;
988         struct hwi_controller *phwi_ctrlr;
989         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
990
991         phwi_ctrlr = phba->phwi_ctrlr;
992         pwrb_context = &phwi_ctrlr->wrb_context[cid];
993         if (pwrb_context->wrb_handles_available >= 2) {
994                 pwrb_handle = pwrb_context->pwrb_handle_base[
995                                             pwrb_context->alloc_index];
996                 pwrb_context->wrb_handles_available--;
997                 if (pwrb_context->alloc_index ==
998                                                 (phba->params.wrbs_per_cxn - 1))
999                         pwrb_context->alloc_index = 0;
1000                 else
1001                         pwrb_context->alloc_index++;
1002                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1003                                                 pwrb_context->alloc_index];
1004                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1005         } else
1006                 pwrb_handle = NULL;
1007         return pwrb_handle;
1008 }
1009
1010 /**
1011  * free_wrb_handle - To free the wrb handle back to pool
1012  * @phba: The hba pointer
1013  * @pwrb_context: The context to free from
1014  * @pwrb_handle: The wrb_handle to free
1015  *
1016  * This happens under session_lock until submission to chip
1017  */
1018 static void
1019 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1020                 struct wrb_handle *pwrb_handle)
1021 {
1022         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1023         pwrb_context->wrb_handles_available++;
1024         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1025                 pwrb_context->free_index = 0;
1026         else
1027                 pwrb_context->free_index++;
1028
1029         SE_DEBUG(DBG_LVL_8,
1030                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
1031                  "wrb_handles_available=%d\n",
1032                  pwrb_handle, pwrb_context->free_index,
1033                  pwrb_context->wrb_handles_available);
1034 }
1035
1036 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1037 {
1038         struct sgl_handle *psgl_handle;
1039
1040         if (phba->eh_sgl_hndl_avbl) {
1041                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1042                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1043                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
1044                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1045                 phba->eh_sgl_hndl_avbl--;
1046                 if (phba->eh_sgl_alloc_index ==
1047                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1048                      1))
1049                         phba->eh_sgl_alloc_index = 0;
1050                 else
1051                         phba->eh_sgl_alloc_index++;
1052         } else
1053                 psgl_handle = NULL;
1054         return psgl_handle;
1055 }
1056
1057 void
1058 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1059 {
1060
1061         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1062                              phba->eh_sgl_free_index);
1063         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1064                 /*
1065                  * this can happen if clean_task is called on a task that
1066                  * failed in xmit_task or alloc_pdu.
1067                  */
1068                 SE_DEBUG(DBG_LVL_8,
1069                          "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1070                          phba->eh_sgl_free_index);
1071                 return;
1072         }
1073         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1074         phba->eh_sgl_hndl_avbl++;
1075         if (phba->eh_sgl_free_index ==
1076             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1077                 phba->eh_sgl_free_index = 0;
1078         else
1079                 phba->eh_sgl_free_index++;
1080 }
1081
1082 static void
1083 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1084                struct iscsi_task *task, struct sol_cqe *psol)
1085 {
1086         struct beiscsi_io_task *io_task = task->dd_data;
1087         struct be_status_bhs *sts_bhs =
1088                                 (struct be_status_bhs *)io_task->cmd_bhs;
1089         struct iscsi_conn *conn = beiscsi_conn->conn;
1090         unsigned int sense_len;
1091         unsigned char *sense;
1092         u32 resid = 0, exp_cmdsn, max_cmdsn;
1093         u8 rsp, status, flags;
1094
1095         exp_cmdsn = (psol->
1096                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1097                         & SOL_EXP_CMD_SN_MASK);
1098         max_cmdsn = ((psol->
1099                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1100                         & SOL_EXP_CMD_SN_MASK) +
1101                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1102                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1103         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1104                                                 & SOL_RESP_MASK) >> 16);
1105         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1106                                                 & SOL_STS_MASK) >> 8);
1107         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1108                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1109
1110         task->sc->result = (DID_OK << 16) | status;
1111         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1112                 task->sc->result = DID_ERROR << 16;
1113                 goto unmap;
1114         }
1115
1116         /* bidi not initially supported */
1117         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1118                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1119                                 32] & SOL_RES_CNT_MASK);
1120
1121                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1122                         task->sc->result = DID_ERROR << 16;
1123
1124                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1125                         scsi_set_resid(task->sc, resid);
1126                         if (!status && (scsi_bufflen(task->sc) - resid <
1127                             task->sc->underflow))
1128                                 task->sc->result = DID_ERROR << 16;
1129                 }
1130         }
1131
1132         if (status == SAM_STAT_CHECK_CONDITION) {
1133                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1134                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1135                 sense_len =  cpu_to_be16(*slen);
1136                 memcpy(task->sc->sense_buffer, sense,
1137                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1138         }
1139
1140         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1141                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1142                                                         & SOL_RES_CNT_MASK)
1143                          conn->rxdata_octets += (psol->
1144                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1145                              & SOL_RES_CNT_MASK);
1146         }
1147 unmap:
1148         scsi_dma_unmap(io_task->scsi_cmnd);
1149         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1150 }
1151
1152 static void
1153 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1154                    struct iscsi_task *task, struct sol_cqe *psol)
1155 {
1156         struct iscsi_logout_rsp *hdr;
1157         struct beiscsi_io_task *io_task = task->dd_data;
1158         struct iscsi_conn *conn = beiscsi_conn->conn;
1159
1160         hdr = (struct iscsi_logout_rsp *)task->hdr;
1161         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1162         hdr->t2wait = 5;
1163         hdr->t2retain = 0;
1164         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1165                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1166         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1167                                         32] & SOL_RESP_MASK);
1168         hdr->exp_cmdsn = cpu_to_be32(psol->
1169                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1170                                         & SOL_EXP_CMD_SN_MASK);
1171         hdr->max_cmdsn = be32_to_cpu((psol->
1172                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1173                                         & SOL_EXP_CMD_SN_MASK) +
1174                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1175                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1176         hdr->dlength[0] = 0;
1177         hdr->dlength[1] = 0;
1178         hdr->dlength[2] = 0;
1179         hdr->hlength = 0;
1180         hdr->itt = io_task->libiscsi_itt;
1181         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1182 }
1183
1184 static void
1185 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1186                 struct iscsi_task *task, struct sol_cqe *psol)
1187 {
1188         struct iscsi_tm_rsp *hdr;
1189         struct iscsi_conn *conn = beiscsi_conn->conn;
1190         struct beiscsi_io_task *io_task = task->dd_data;
1191
1192         hdr = (struct iscsi_tm_rsp *)task->hdr;
1193         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1194         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1195                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1196         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1197                                         32] & SOL_RESP_MASK);
1198         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1199                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1200         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1201                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1202                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1203                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1204         hdr->itt = io_task->libiscsi_itt;
1205         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1206 }
1207
1208 static void
1209 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1210                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1211 {
1212         struct hwi_wrb_context *pwrb_context;
1213         struct wrb_handle *pwrb_handle = NULL;
1214         struct hwi_controller *phwi_ctrlr;
1215         struct iscsi_task *task;
1216         struct beiscsi_io_task *io_task;
1217         struct iscsi_conn *conn = beiscsi_conn->conn;
1218         struct iscsi_session *session = conn->session;
1219
1220         phwi_ctrlr = phba->phwi_ctrlr;
1221         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1222                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1223                                 SOL_CID_MASK) >> 6) -
1224                                 phba->fw_config.iscsi_cid_start];
1225         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1226                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1227                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1228         task = pwrb_handle->pio_handle;
1229
1230         io_task = task->dd_data;
1231         spin_lock(&phba->mgmt_sgl_lock);
1232         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1233         spin_unlock(&phba->mgmt_sgl_lock);
1234         spin_lock_bh(&session->lock);
1235         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1236         spin_unlock_bh(&session->lock);
1237 }
1238
1239 static void
1240 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1241                        struct iscsi_task *task, struct sol_cqe *psol)
1242 {
1243         struct iscsi_nopin *hdr;
1244         struct iscsi_conn *conn = beiscsi_conn->conn;
1245         struct beiscsi_io_task *io_task = task->dd_data;
1246
1247         hdr = (struct iscsi_nopin *)task->hdr;
1248         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1249                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1250         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1251                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1252         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1253                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1254                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1255                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1256         hdr->opcode = ISCSI_OP_NOOP_IN;
1257         hdr->itt = io_task->libiscsi_itt;
1258         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1259 }
1260
1261 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1262                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1263 {
1264         struct hwi_wrb_context *pwrb_context;
1265         struct wrb_handle *pwrb_handle;
1266         struct iscsi_wrb *pwrb = NULL;
1267         struct hwi_controller *phwi_ctrlr;
1268         struct iscsi_task *task;
1269         unsigned int type;
1270         struct iscsi_conn *conn = beiscsi_conn->conn;
1271         struct iscsi_session *session = conn->session;
1272
1273         phwi_ctrlr = phba->phwi_ctrlr;
1274         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1275                                 (struct amap_sol_cqe, cid) / 32]
1276                                 & SOL_CID_MASK) >> 6) -
1277                                 phba->fw_config.iscsi_cid_start];
1278         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1279                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1280                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1281         task = pwrb_handle->pio_handle;
1282         pwrb = pwrb_handle->pwrb;
1283         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1284                                  WRB_TYPE_MASK) >> 28;
1285
1286         spin_lock_bh(&session->lock);
1287         switch (type) {
1288         case HWH_TYPE_IO:
1289         case HWH_TYPE_IO_RD:
1290                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1291                      ISCSI_OP_NOOP_OUT)
1292                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1293                 else
1294                         be_complete_io(beiscsi_conn, task, psol);
1295                 break;
1296
1297         case HWH_TYPE_LOGOUT:
1298                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1299                         be_complete_logout(beiscsi_conn, task, psol);
1300                 else
1301                         be_complete_tmf(beiscsi_conn, task, psol);
1302
1303                 break;
1304
1305         case HWH_TYPE_LOGIN:
1306                 SE_DEBUG(DBG_LVL_1,
1307                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1308                          "- Solicited path\n");
1309                 break;
1310
1311         case HWH_TYPE_NOP:
1312                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1313                 break;
1314
1315         default:
1316                 shost_printk(KERN_WARNING, phba->shost,
1317                                 "In hwi_complete_cmd, unknown type = %d"
1318                                 "wrb_index 0x%x CID 0x%x\n", type,
1319                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1320                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1321                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1322                                 cid) / 32] & SOL_CID_MASK) >> 6));
1323                 break;
1324         }
1325
1326         spin_unlock_bh(&session->lock);
1327 }
1328
1329 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1330                                           *pasync_ctx, unsigned int is_header,
1331                                           unsigned int host_write_ptr)
1332 {
1333         if (is_header)
1334                 return &pasync_ctx->async_entry[host_write_ptr].
1335                     header_busy_list;
1336         else
1337                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1338 }
1339
1340 static struct async_pdu_handle *
1341 hwi_get_async_handle(struct beiscsi_hba *phba,
1342                      struct beiscsi_conn *beiscsi_conn,
1343                      struct hwi_async_pdu_context *pasync_ctx,
1344                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1345 {
1346         struct be_bus_address phys_addr;
1347         struct list_head *pbusy_list;
1348         struct async_pdu_handle *pasync_handle = NULL;
1349         int buffer_len = 0;
1350         unsigned char buffer_index = -1;
1351         unsigned char is_header = 0;
1352
1353         phys_addr.u.a32.address_lo =
1354             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1355             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1356                                                 & PDUCQE_DPL_MASK) >> 16);
1357         phys_addr.u.a32.address_hi =
1358             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1359
1360         phys_addr.u.a64.address =
1361                         *((unsigned long long *)(&phys_addr.u.a64.address));
1362
1363         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1364                         & PDUCQE_CODE_MASK) {
1365         case UNSOL_HDR_NOTIFY:
1366                 is_header = 1;
1367
1368                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1369                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1370                         index) / 32] & PDUCQE_INDEX_MASK));
1371
1372                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1373                                 pasync_ctx->async_header.pa_base.u.a64.address);
1374
1375                 buffer_index = buffer_len /
1376                                 pasync_ctx->async_header.buffer_size;
1377
1378                 break;
1379         case UNSOL_DATA_NOTIFY:
1380                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1381                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1382                                         index) / 32] & PDUCQE_INDEX_MASK));
1383                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1384                                         pasync_ctx->async_data.pa_base.u.
1385                                         a64.address);
1386                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1387                 break;
1388         default:
1389                 pbusy_list = NULL;
1390                 shost_printk(KERN_WARNING, phba->shost,
1391                         "Unexpected code=%d\n",
1392                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1393                                         code) / 32] & PDUCQE_CODE_MASK);
1394                 return NULL;
1395         }
1396
1397         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1398         WARN_ON(list_empty(pbusy_list));
1399         list_for_each_entry(pasync_handle, pbusy_list, link) {
1400                 WARN_ON(pasync_handle->consumed);
1401                 if (pasync_handle->index == buffer_index)
1402                         break;
1403         }
1404
1405         WARN_ON(!pasync_handle);
1406
1407         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1408                                              phba->fw_config.iscsi_cid_start;
1409         pasync_handle->is_header = is_header;
1410         pasync_handle->buffer_len = ((pdpdu_cqe->
1411                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1412                         & PDUCQE_DPL_MASK) >> 16);
1413
1414         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1415                         index) / 32] & PDUCQE_INDEX_MASK);
1416         return pasync_handle;
1417 }
1418
1419 static unsigned int
1420 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1421                            unsigned int is_header, unsigned int cq_index)
1422 {
1423         struct list_head *pbusy_list;
1424         struct async_pdu_handle *pasync_handle;
1425         unsigned int num_entries, writables = 0;
1426         unsigned int *pep_read_ptr, *pwritables;
1427
1428
1429         if (is_header) {
1430                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1431                 pwritables = &pasync_ctx->async_header.writables;
1432                 num_entries = pasync_ctx->async_header.num_entries;
1433         } else {
1434                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1435                 pwritables = &pasync_ctx->async_data.writables;
1436                 num_entries = pasync_ctx->async_data.num_entries;
1437         }
1438
1439         while ((*pep_read_ptr) != cq_index) {
1440                 (*pep_read_ptr)++;
1441                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1442
1443                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1444                                                      *pep_read_ptr);
1445                 if (writables == 0)
1446                         WARN_ON(list_empty(pbusy_list));
1447
1448                 if (!list_empty(pbusy_list)) {
1449                         pasync_handle = list_entry(pbusy_list->next,
1450                                                    struct async_pdu_handle,
1451                                                    link);
1452                         WARN_ON(!pasync_handle);
1453                         pasync_handle->consumed = 1;
1454                 }
1455
1456                 writables++;
1457         }
1458
1459         if (!writables) {
1460                 SE_DEBUG(DBG_LVL_1,
1461                          "Duplicate notification received - index 0x%x!!\n",
1462                          cq_index);
1463                 WARN_ON(1);
1464         }
1465
1466         *pwritables = *pwritables + writables;
1467         return 0;
1468 }
1469
1470 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1471                                        unsigned int cri)
1472 {
1473         struct hwi_controller *phwi_ctrlr;
1474         struct hwi_async_pdu_context *pasync_ctx;
1475         struct async_pdu_handle *pasync_handle, *tmp_handle;
1476         struct list_head *plist;
1477         unsigned int i = 0;
1478
1479         phwi_ctrlr = phba->phwi_ctrlr;
1480         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1481
1482         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1483
1484         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1485                 list_del(&pasync_handle->link);
1486
1487                 if (i == 0) {
1488                         list_add_tail(&pasync_handle->link,
1489                                       &pasync_ctx->async_header.free_list);
1490                         pasync_ctx->async_header.free_entries++;
1491                         i++;
1492                 } else {
1493                         list_add_tail(&pasync_handle->link,
1494                                       &pasync_ctx->async_data.free_list);
1495                         pasync_ctx->async_data.free_entries++;
1496                         i++;
1497                 }
1498         }
1499
1500         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1501         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1502         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1503         return 0;
1504 }
1505
1506 static struct phys_addr *
1507 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1508                      unsigned int is_header, unsigned int host_write_ptr)
1509 {
1510         struct phys_addr *pasync_sge = NULL;
1511
1512         if (is_header)
1513                 pasync_sge = pasync_ctx->async_header.ring_base;
1514         else
1515                 pasync_sge = pasync_ctx->async_data.ring_base;
1516
1517         return pasync_sge + host_write_ptr;
1518 }
1519
1520 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1521                                    unsigned int is_header)
1522 {
1523         struct hwi_controller *phwi_ctrlr;
1524         struct hwi_async_pdu_context *pasync_ctx;
1525         struct async_pdu_handle *pasync_handle;
1526         struct list_head *pfree_link, *pbusy_list;
1527         struct phys_addr *pasync_sge;
1528         unsigned int ring_id, num_entries;
1529         unsigned int host_write_num;
1530         unsigned int writables;
1531         unsigned int i = 0;
1532         u32 doorbell = 0;
1533
1534         phwi_ctrlr = phba->phwi_ctrlr;
1535         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1536
1537         if (is_header) {
1538                 num_entries = pasync_ctx->async_header.num_entries;
1539                 writables = min(pasync_ctx->async_header.writables,
1540                                 pasync_ctx->async_header.free_entries);
1541                 pfree_link = pasync_ctx->async_header.free_list.next;
1542                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1543                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1544         } else {
1545                 num_entries = pasync_ctx->async_data.num_entries;
1546                 writables = min(pasync_ctx->async_data.writables,
1547                                 pasync_ctx->async_data.free_entries);
1548                 pfree_link = pasync_ctx->async_data.free_list.next;
1549                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1550                 ring_id = phwi_ctrlr->default_pdu_data.id;
1551         }
1552
1553         writables = (writables / 8) * 8;
1554         if (writables) {
1555                 for (i = 0; i < writables; i++) {
1556                         pbusy_list =
1557                             hwi_get_async_busy_list(pasync_ctx, is_header,
1558                                                     host_write_num);
1559                         pasync_handle =
1560                             list_entry(pfree_link, struct async_pdu_handle,
1561                                                                 link);
1562                         WARN_ON(!pasync_handle);
1563                         pasync_handle->consumed = 0;
1564
1565                         pfree_link = pfree_link->next;
1566
1567                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1568                                                 is_header, host_write_num);
1569
1570                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1571                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1572
1573                         list_move(&pasync_handle->link, pbusy_list);
1574
1575                         host_write_num++;
1576                         host_write_num = host_write_num % num_entries;
1577                 }
1578
1579                 if (is_header) {
1580                         pasync_ctx->async_header.host_write_ptr =
1581                                                         host_write_num;
1582                         pasync_ctx->async_header.free_entries -= writables;
1583                         pasync_ctx->async_header.writables -= writables;
1584                         pasync_ctx->async_header.busy_entries += writables;
1585                 } else {
1586                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1587                         pasync_ctx->async_data.free_entries -= writables;
1588                         pasync_ctx->async_data.writables -= writables;
1589                         pasync_ctx->async_data.busy_entries += writables;
1590                 }
1591
1592                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1593                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1594                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1595                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1596                                         << DB_DEF_PDU_CQPROC_SHIFT;
1597
1598                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1599         }
1600 }
1601
1602 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1603                                          struct beiscsi_conn *beiscsi_conn,
1604                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1605 {
1606         struct hwi_controller *phwi_ctrlr;
1607         struct hwi_async_pdu_context *pasync_ctx;
1608         struct async_pdu_handle *pasync_handle = NULL;
1609         unsigned int cq_index = -1;
1610
1611         phwi_ctrlr = phba->phwi_ctrlr;
1612         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1613
1614         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1615                                              pdpdu_cqe, &cq_index);
1616         BUG_ON(pasync_handle->is_header != 0);
1617         if (pasync_handle->consumed == 0)
1618                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1619                                            cq_index);
1620
1621         hwi_free_async_msg(phba, pasync_handle->cri);
1622         hwi_post_async_buffers(phba, pasync_handle->is_header);
1623 }
1624
1625 static unsigned int
1626 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1627                   struct beiscsi_hba *phba,
1628                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1629 {
1630         struct list_head *plist;
1631         struct async_pdu_handle *pasync_handle;
1632         void *phdr = NULL;
1633         unsigned int hdr_len = 0, buf_len = 0;
1634         unsigned int status, index = 0, offset = 0;
1635         void *pfirst_buffer = NULL;
1636         unsigned int num_buf = 0;
1637
1638         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1639
1640         list_for_each_entry(pasync_handle, plist, link) {
1641                 if (index == 0) {
1642                         phdr = pasync_handle->pbuffer;
1643                         hdr_len = pasync_handle->buffer_len;
1644                 } else {
1645                         buf_len = pasync_handle->buffer_len;
1646                         if (!num_buf) {
1647                                 pfirst_buffer = pasync_handle->pbuffer;
1648                                 num_buf++;
1649                         }
1650                         memcpy(pfirst_buffer + offset,
1651                                pasync_handle->pbuffer, buf_len);
1652                         offset = buf_len;
1653                 }
1654                 index++;
1655         }
1656
1657         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1658                                            (beiscsi_conn->beiscsi_conn_cid -
1659                                             phba->fw_config.iscsi_cid_start),
1660                                             phdr, hdr_len, pfirst_buffer,
1661                                             buf_len);
1662
1663         if (status == 0)
1664                 hwi_free_async_msg(phba, cri);
1665         return 0;
1666 }
1667
1668 static unsigned int
1669 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1670                      struct beiscsi_hba *phba,
1671                      struct async_pdu_handle *pasync_handle)
1672 {
1673         struct hwi_async_pdu_context *pasync_ctx;
1674         struct hwi_controller *phwi_ctrlr;
1675         unsigned int bytes_needed = 0, status = 0;
1676         unsigned short cri = pasync_handle->cri;
1677         struct pdu_base *ppdu;
1678
1679         phwi_ctrlr = phba->phwi_ctrlr;
1680         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1681
1682         list_del(&pasync_handle->link);
1683         if (pasync_handle->is_header) {
1684                 pasync_ctx->async_header.busy_entries--;
1685                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1686                         hwi_free_async_msg(phba, cri);
1687                         BUG();
1688                 }
1689
1690                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1691                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1692                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1693                                 (unsigned short)pasync_handle->buffer_len;
1694                 list_add_tail(&pasync_handle->link,
1695                               &pasync_ctx->async_entry[cri].wait_queue.list);
1696
1697                 ppdu = pasync_handle->pbuffer;
1698                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1699                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1700                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1701                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1702                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1703
1704                 if (status == 0) {
1705                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1706                             bytes_needed;
1707
1708                         if (bytes_needed == 0)
1709                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1710                                                            pasync_ctx, cri);
1711                 }
1712         } else {
1713                 pasync_ctx->async_data.busy_entries--;
1714                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1715                         list_add_tail(&pasync_handle->link,
1716                                       &pasync_ctx->async_entry[cri].wait_queue.
1717                                       list);
1718                         pasync_ctx->async_entry[cri].wait_queue.
1719                                 bytes_received +=
1720                                 (unsigned short)pasync_handle->buffer_len;
1721
1722                         if (pasync_ctx->async_entry[cri].wait_queue.
1723                             bytes_received >=
1724                             pasync_ctx->async_entry[cri].wait_queue.
1725                             bytes_needed)
1726                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1727                                                            pasync_ctx, cri);
1728                 }
1729         }
1730         return status;
1731 }
1732
1733 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1734                                          struct beiscsi_hba *phba,
1735                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1736 {
1737         struct hwi_controller *phwi_ctrlr;
1738         struct hwi_async_pdu_context *pasync_ctx;
1739         struct async_pdu_handle *pasync_handle = NULL;
1740         unsigned int cq_index = -1;
1741
1742         phwi_ctrlr = phba->phwi_ctrlr;
1743         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1744         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1745                                              pdpdu_cqe, &cq_index);
1746
1747         if (pasync_handle->consumed == 0)
1748                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1749                                            cq_index);
1750         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1751         hwi_post_async_buffers(phba, pasync_handle->is_header);
1752 }
1753
1754 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1755 {
1756         struct be_queue_info *mcc_cq;
1757         struct  be_mcc_compl *mcc_compl;
1758         unsigned int num_processed = 0;
1759
1760         mcc_cq = &phba->ctrl.mcc_obj.cq;
1761         mcc_compl = queue_tail_node(mcc_cq);
1762         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1763         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1764
1765                 if (num_processed >= 32) {
1766                         hwi_ring_cq_db(phba, mcc_cq->id,
1767                                         num_processed, 0, 0);
1768                         num_processed = 0;
1769                 }
1770                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1771                         /* Interpret flags as an async trailer */
1772                         if (is_link_state_evt(mcc_compl->flags))
1773                                 /* Interpret compl as a async link evt */
1774                                 beiscsi_async_link_state_process(phba,
1775                                 (struct be_async_event_link_state *) mcc_compl);
1776                         else
1777                                 SE_DEBUG(DBG_LVL_1,
1778                                         " Unsupported Async Event, flags"
1779                                         " = 0x%08x\n", mcc_compl->flags);
1780                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1781                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1782                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1783                 }
1784
1785                 mcc_compl->flags = 0;
1786                 queue_tail_inc(mcc_cq);
1787                 mcc_compl = queue_tail_node(mcc_cq);
1788                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1789                 num_processed++;
1790         }
1791
1792         if (num_processed > 0)
1793                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1794
1795 }
1796
1797 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1798 {
1799         struct be_queue_info *cq;
1800         struct sol_cqe *sol;
1801         struct dmsg_cqe *dmsg;
1802         unsigned int num_processed = 0;
1803         unsigned int tot_nump = 0;
1804         struct beiscsi_conn *beiscsi_conn;
1805         struct beiscsi_endpoint *beiscsi_ep;
1806         struct iscsi_endpoint *ep;
1807         struct beiscsi_hba *phba;
1808
1809         cq = pbe_eq->cq;
1810         sol = queue_tail_node(cq);
1811         phba = pbe_eq->phba;
1812
1813         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1814                CQE_VALID_MASK) {
1815                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1816
1817                 ep = phba->ep_array[(u32) ((sol->
1818                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1819                                    SOL_CID_MASK) >> 6) -
1820                                    phba->fw_config.iscsi_cid_start];
1821
1822                 beiscsi_ep = ep->dd_data;
1823                 beiscsi_conn = beiscsi_ep->conn;
1824
1825                 if (num_processed >= 32) {
1826                         hwi_ring_cq_db(phba, cq->id,
1827                                         num_processed, 0, 0);
1828                         tot_nump += num_processed;
1829                         num_processed = 0;
1830                 }
1831
1832                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1833                         32] & CQE_CODE_MASK) {
1834                 case SOL_CMD_COMPLETE:
1835                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1836                         break;
1837                 case DRIVERMSG_NOTIFY:
1838                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
1839                         dmsg = (struct dmsg_cqe *)sol;
1840                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1841                         break;
1842                 case UNSOL_HDR_NOTIFY:
1843                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1844                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1845                                              (struct i_t_dpdu_cqe *)sol);
1846                         break;
1847                 case UNSOL_DATA_NOTIFY:
1848                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1849                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1850                                              (struct i_t_dpdu_cqe *)sol);
1851                         break;
1852                 case CXN_INVALIDATE_INDEX_NOTIFY:
1853                 case CMD_INVALIDATED_NOTIFY:
1854                 case CXN_INVALIDATE_NOTIFY:
1855                         SE_DEBUG(DBG_LVL_1,
1856                                  "Ignoring CQ Error notification for cmd/cxn"
1857                                  "invalidate\n");
1858                         break;
1859                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1860                 case CMD_KILLED_INVALID_STATSN_RCVD:
1861                 case CMD_KILLED_INVALID_R2T_RCVD:
1862                 case CMD_CXN_KILLED_LUN_INVALID:
1863                 case CMD_CXN_KILLED_ICD_INVALID:
1864                 case CMD_CXN_KILLED_ITT_INVALID:
1865                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1866                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1867                         SE_DEBUG(DBG_LVL_1,
1868                                  "CQ Error notification for cmd.. "
1869                                  "code %d cid 0x%x\n",
1870                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1871                                  32] & CQE_CODE_MASK,
1872                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1873                                  32] & SOL_CID_MASK));
1874                         break;
1875                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1876                         SE_DEBUG(DBG_LVL_1,
1877                                  "Digest error on def pdu ring, dropping..\n");
1878                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1879                                              (struct i_t_dpdu_cqe *) sol);
1880                         break;
1881                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1882                 case CXN_KILLED_BURST_LEN_MISMATCH:
1883                 case CXN_KILLED_AHS_RCVD:
1884                 case CXN_KILLED_HDR_DIGEST_ERR:
1885                 case CXN_KILLED_UNKNOWN_HDR:
1886                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1887                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1888                 case CXN_KILLED_TIMED_OUT:
1889                 case CXN_KILLED_FIN_RCVD:
1890                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1891                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1892                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1893                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1894                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1895                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1896                                  "0x%x...\n",
1897                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1898                                  32] & CQE_CODE_MASK,
1899                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1900                                  32] & CQE_CID_MASK));
1901                         iscsi_conn_failure(beiscsi_conn->conn,
1902                                            ISCSI_ERR_CONN_FAILED);
1903                         break;
1904                 case CXN_KILLED_RST_SENT:
1905                 case CXN_KILLED_RST_RCVD:
1906                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1907                                 "received/sent on CID 0x%x...\n",
1908                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1909                                  32] & CQE_CODE_MASK,
1910                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1911                                  32] & CQE_CID_MASK));
1912                         iscsi_conn_failure(beiscsi_conn->conn,
1913                                            ISCSI_ERR_CONN_FAILED);
1914                         break;
1915                 default:
1916                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1917                                  "received on CID 0x%x...\n",
1918                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1919                                  32] & CQE_CODE_MASK,
1920                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1921                                  32] & CQE_CID_MASK));
1922                         break;
1923                 }
1924
1925                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1926                 queue_tail_inc(cq);
1927                 sol = queue_tail_node(cq);
1928                 num_processed++;
1929         }
1930
1931         if (num_processed > 0) {
1932                 tot_nump += num_processed;
1933                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1934         }
1935         return tot_nump;
1936 }
1937
1938 void beiscsi_process_all_cqs(struct work_struct *work)
1939 {
1940         unsigned long flags;
1941         struct hwi_controller *phwi_ctrlr;
1942         struct hwi_context_memory *phwi_context;
1943         struct be_eq_obj *pbe_eq;
1944         struct beiscsi_hba *phba =
1945             container_of(work, struct beiscsi_hba, work_cqs);
1946
1947         phwi_ctrlr = phba->phwi_ctrlr;
1948         phwi_context = phwi_ctrlr->phwi_ctxt;
1949         if (phba->msix_enabled)
1950                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1951         else
1952                 pbe_eq = &phwi_context->be_eq[0];
1953
1954         if (phba->todo_mcc_cq) {
1955                 spin_lock_irqsave(&phba->isr_lock, flags);
1956                 phba->todo_mcc_cq = 0;
1957                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1958                 beiscsi_process_mcc_isr(phba);
1959         }
1960
1961         if (phba->todo_cq) {
1962                 spin_lock_irqsave(&phba->isr_lock, flags);
1963                 phba->todo_cq = 0;
1964                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1965                 beiscsi_process_cq(pbe_eq);
1966         }
1967 }
1968
1969 static int be_iopoll(struct blk_iopoll *iop, int budget)
1970 {
1971         static unsigned int ret;
1972         struct beiscsi_hba *phba;
1973         struct be_eq_obj *pbe_eq;
1974
1975         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1976         ret = beiscsi_process_cq(pbe_eq);
1977         if (ret < budget) {
1978                 phba = pbe_eq->phba;
1979                 blk_iopoll_complete(iop);
1980                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1981                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1982         }
1983         return ret;
1984 }
1985
1986 static void
1987 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1988               unsigned int num_sg, struct beiscsi_io_task *io_task)
1989 {
1990         struct iscsi_sge *psgl;
1991         unsigned int sg_len, index;
1992         unsigned int sge_len = 0;
1993         unsigned long long addr;
1994         struct scatterlist *l_sg;
1995         unsigned int offset;
1996
1997         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1998                                       io_task->bhs_pa.u.a32.address_lo);
1999         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2000                                       io_task->bhs_pa.u.a32.address_hi);
2001
2002         l_sg = sg;
2003         for (index = 0; (index < num_sg) && (index < 2); index++,
2004                                                          sg = sg_next(sg)) {
2005                 if (index == 0) {
2006                         sg_len = sg_dma_len(sg);
2007                         addr = (u64) sg_dma_address(sg);
2008                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2009                                                 ((u32)(addr & 0xFFFFFFFF)));
2010                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2011                                                         ((u32)(addr >> 32)));
2012                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2013                                                         sg_len);
2014                         sge_len = sg_len;
2015                 } else {
2016                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2017                                                         pwrb, sge_len);
2018                         sg_len = sg_dma_len(sg);
2019                         addr = (u64) sg_dma_address(sg);
2020                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2021                                                 ((u32)(addr & 0xFFFFFFFF)));
2022                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2023                                                         ((u32)(addr >> 32)));
2024                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2025                                                         sg_len);
2026                 }
2027         }
2028         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2029         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2030
2031         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2032
2033         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2034                         io_task->bhs_pa.u.a32.address_hi);
2035         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2036                         io_task->bhs_pa.u.a32.address_lo);
2037
2038         if (num_sg == 1) {
2039                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2040                                                                 1);
2041                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2042                                                                 0);
2043         } else if (num_sg == 2) {
2044                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2045                                                                 0);
2046                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2047                                                                 1);
2048         } else {
2049                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2050                                                                 0);
2051                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2052                                                                 0);
2053         }
2054         sg = l_sg;
2055         psgl++;
2056         psgl++;
2057         offset = 0;
2058         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2059                 sg_len = sg_dma_len(sg);
2060                 addr = (u64) sg_dma_address(sg);
2061                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2062                                                 (addr & 0xFFFFFFFF));
2063                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2064                                                 (addr >> 32));
2065                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2066                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2067                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2068                 offset += sg_len;
2069         }
2070         psgl--;
2071         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2072 }
2073
2074 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2075 {
2076         struct iscsi_sge *psgl;
2077         unsigned long long addr;
2078         struct beiscsi_io_task *io_task = task->dd_data;
2079         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2080         struct beiscsi_hba *phba = beiscsi_conn->phba;
2081
2082         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2083         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2084                                 io_task->bhs_pa.u.a32.address_lo);
2085         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2086                                 io_task->bhs_pa.u.a32.address_hi);
2087
2088         if (task->data) {
2089                 if (task->data_count) {
2090                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2091                         addr = (u64) pci_map_single(phba->pcidev,
2092                                                     task->data,
2093                                                     task->data_count, 1);
2094                 } else {
2095                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2096                         addr = 0;
2097                 }
2098                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2099                                                 ((u32)(addr & 0xFFFFFFFF)));
2100                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2101                                                 ((u32)(addr >> 32)));
2102                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2103                                                 task->data_count);
2104
2105                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2106         } else {
2107                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2108                 addr = 0;
2109         }
2110
2111         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2112
2113         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2114
2115         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2116                       io_task->bhs_pa.u.a32.address_hi);
2117         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2118                       io_task->bhs_pa.u.a32.address_lo);
2119         if (task->data) {
2120                 psgl++;
2121                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2122                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2123                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2124                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2125                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2126                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2127
2128                 psgl++;
2129                 if (task->data) {
2130                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2131                                                 ((u32)(addr & 0xFFFFFFFF)));
2132                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2133                                                 ((u32)(addr >> 32)));
2134                 }
2135                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2136         }
2137         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2138 }
2139
2140 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2141 {
2142         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2143         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2144         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2145
2146         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2147                                       sizeof(struct sol_cqe));
2148         num_async_pdu_buf_pages =
2149                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2150                                        phba->params.defpdu_hdr_sz);
2151         num_async_pdu_buf_sgl_pages =
2152                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2153                                        sizeof(struct phys_addr));
2154         num_async_pdu_data_pages =
2155                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2156                                        phba->params.defpdu_data_sz);
2157         num_async_pdu_data_sgl_pages =
2158                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2159                                        sizeof(struct phys_addr));
2160
2161         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2162
2163         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2164                                                  BE_ISCSI_PDU_HEADER_SIZE;
2165         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2166                                             sizeof(struct hwi_context_memory);
2167
2168
2169         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2170             * (phba->params.wrbs_per_cxn)
2171             * phba->params.cxns_per_ctrl;
2172         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2173                                  (phba->params.wrbs_per_cxn);
2174         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2175                                 phba->params.cxns_per_ctrl);
2176
2177         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2178                 phba->params.icds_per_ctrl;
2179         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2180                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2181
2182         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2183                 num_async_pdu_buf_pages * PAGE_SIZE;
2184         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2185                 num_async_pdu_data_pages * PAGE_SIZE;
2186         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2187                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2188         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2189                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2190         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2191                 phba->params.asyncpdus_per_ctrl *
2192                 sizeof(struct async_pdu_handle);
2193         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2194                 phba->params.asyncpdus_per_ctrl *
2195                 sizeof(struct async_pdu_handle);
2196         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2197                 sizeof(struct hwi_async_pdu_context) +
2198                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2199 }
2200
2201 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2202 {
2203         struct be_mem_descriptor *mem_descr;
2204         dma_addr_t bus_add;
2205         struct mem_array *mem_arr, *mem_arr_orig;
2206         unsigned int i, j, alloc_size, curr_alloc_size;
2207
2208         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2209         if (!phba->phwi_ctrlr)
2210                 return -ENOMEM;
2211
2212         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2213                                  GFP_KERNEL);
2214         if (!phba->init_mem) {
2215                 kfree(phba->phwi_ctrlr);
2216                 return -ENOMEM;
2217         }
2218
2219         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2220                                GFP_KERNEL);
2221         if (!mem_arr_orig) {
2222                 kfree(phba->init_mem);
2223                 kfree(phba->phwi_ctrlr);
2224                 return -ENOMEM;
2225         }
2226
2227         mem_descr = phba->init_mem;
2228         for (i = 0; i < SE_MEM_MAX; i++) {
2229                 j = 0;
2230                 mem_arr = mem_arr_orig;
2231                 alloc_size = phba->mem_req[i];
2232                 memset(mem_arr, 0, sizeof(struct mem_array) *
2233                        BEISCSI_MAX_FRAGS_INIT);
2234                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2235                 do {
2236                         mem_arr->virtual_address = pci_alloc_consistent(
2237                                                         phba->pcidev,
2238                                                         curr_alloc_size,
2239                                                         &bus_add);
2240                         if (!mem_arr->virtual_address) {
2241                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2242                                         goto free_mem;
2243                                 if (curr_alloc_size -
2244                                         rounddown_pow_of_two(curr_alloc_size))
2245                                         curr_alloc_size = rounddown_pow_of_two
2246                                                              (curr_alloc_size);
2247                                 else
2248                                         curr_alloc_size = curr_alloc_size / 2;
2249                         } else {
2250                                 mem_arr->bus_address.u.
2251                                     a64.address = (__u64) bus_add;
2252                                 mem_arr->size = curr_alloc_size;
2253                                 alloc_size -= curr_alloc_size;
2254                                 curr_alloc_size = min(be_max_phys_size *
2255                                                       1024, alloc_size);
2256                                 j++;
2257                                 mem_arr++;
2258                         }
2259                 } while (alloc_size);
2260                 mem_descr->num_elements = j;
2261                 mem_descr->size_in_bytes = phba->mem_req[i];
2262                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2263                                                GFP_KERNEL);
2264                 if (!mem_descr->mem_array)
2265                         goto free_mem;
2266
2267                 memcpy(mem_descr->mem_array, mem_arr_orig,
2268                        sizeof(struct mem_array) * j);
2269                 mem_descr++;
2270         }
2271         kfree(mem_arr_orig);
2272         return 0;
2273 free_mem:
2274         mem_descr->num_elements = j;
2275         while ((i) || (j)) {
2276                 for (j = mem_descr->num_elements; j > 0; j--) {
2277                         pci_free_consistent(phba->pcidev,
2278                                             mem_descr->mem_array[j - 1].size,
2279                                             mem_descr->mem_array[j - 1].
2280                                             virtual_address,
2281                                             (unsigned long)mem_descr->
2282                                             mem_array[j - 1].
2283                                             bus_address.u.a64.address);
2284                 }
2285                 if (i) {
2286                         i--;
2287                         kfree(mem_descr->mem_array);
2288                         mem_descr--;
2289                 }
2290         }
2291         kfree(mem_arr_orig);
2292         kfree(phba->init_mem);
2293         kfree(phba->phwi_ctrlr);
2294         return -ENOMEM;
2295 }
2296
2297 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2298 {
2299         beiscsi_find_mem_req(phba);
2300         return beiscsi_alloc_mem(phba);
2301 }
2302
2303 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2304 {
2305         struct pdu_data_out *pdata_out;
2306         struct pdu_nop_out *pnop_out;
2307         struct be_mem_descriptor *mem_descr;
2308
2309         mem_descr = phba->init_mem;
2310         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2311         pdata_out =
2312             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2313         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2314
2315         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2316                       IIOC_SCSI_DATA);
2317
2318         pnop_out =
2319             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2320                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2321
2322         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2323         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2324         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2325         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2326 }
2327
2328 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2329 {
2330         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2331         struct wrb_handle *pwrb_handle;
2332         struct hwi_controller *phwi_ctrlr;
2333         struct hwi_wrb_context *pwrb_context;
2334         struct iscsi_wrb *pwrb;
2335         unsigned int num_cxn_wrbh;
2336         unsigned int num_cxn_wrb, j, idx, index;
2337
2338         mem_descr_wrbh = phba->init_mem;
2339         mem_descr_wrbh += HWI_MEM_WRBH;
2340
2341         mem_descr_wrb = phba->init_mem;
2342         mem_descr_wrb += HWI_MEM_WRB;
2343
2344         idx = 0;
2345         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2346         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2347                         ((sizeof(struct wrb_handle)) *
2348                          phba->params.wrbs_per_cxn));
2349         phwi_ctrlr = phba->phwi_ctrlr;
2350
2351         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2352                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2353                 pwrb_context->pwrb_handle_base =
2354                                 kzalloc(sizeof(struct wrb_handle *) *
2355                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2356                 pwrb_context->pwrb_handle_basestd =
2357                                 kzalloc(sizeof(struct wrb_handle *) *
2358                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2359                 if (num_cxn_wrbh) {
2360                         pwrb_context->alloc_index = 0;
2361                         pwrb_context->wrb_handles_available = 0;
2362                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2363                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2364                                 pwrb_context->pwrb_handle_basestd[j] =
2365                                                                 pwrb_handle;
2366                                 pwrb_context->wrb_handles_available++;
2367                                 pwrb_handle->wrb_index = j;
2368                                 pwrb_handle++;
2369                         }
2370                         pwrb_context->free_index = 0;
2371                         num_cxn_wrbh--;
2372                 } else {
2373                         idx++;
2374                         pwrb_handle =
2375                             mem_descr_wrbh->mem_array[idx].virtual_address;
2376                         num_cxn_wrbh =
2377                             ((mem_descr_wrbh->mem_array[idx].size) /
2378                              ((sizeof(struct wrb_handle)) *
2379                               phba->params.wrbs_per_cxn));
2380                         pwrb_context->alloc_index = 0;
2381                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2382                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2383                                 pwrb_context->pwrb_handle_basestd[j] =
2384                                     pwrb_handle;
2385                                 pwrb_context->wrb_handles_available++;
2386                                 pwrb_handle->wrb_index = j;
2387                                 pwrb_handle++;
2388                         }
2389                         pwrb_context->free_index = 0;
2390                         num_cxn_wrbh--;
2391                 }
2392         }
2393         idx = 0;
2394         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2395         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2396                       ((sizeof(struct iscsi_wrb) *
2397                         phba->params.wrbs_per_cxn));
2398         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2399                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2400                 if (num_cxn_wrb) {
2401                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2402                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2403                                 pwrb_handle->pwrb = pwrb;
2404                                 pwrb++;
2405                         }
2406                         num_cxn_wrb--;
2407                 } else {
2408                         idx++;
2409                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2410                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2411                                       ((sizeof(struct iscsi_wrb) *
2412                                         phba->params.wrbs_per_cxn));
2413                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2414                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2415                                 pwrb_handle->pwrb = pwrb;
2416                                 pwrb++;
2417                         }
2418                         num_cxn_wrb--;
2419                 }
2420         }
2421 }
2422
2423 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2424 {
2425         struct hwi_controller *phwi_ctrlr;
2426         struct hba_parameters *p = &phba->params;
2427         struct hwi_async_pdu_context *pasync_ctx;
2428         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2429         unsigned int index;
2430         struct be_mem_descriptor *mem_descr;
2431
2432         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2433         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2434
2435         phwi_ctrlr = phba->phwi_ctrlr;
2436         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2437                                 mem_descr->mem_array[0].virtual_address;
2438         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2439         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2440
2441         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2442         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2443         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2444         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2445
2446         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2447         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2448         if (mem_descr->mem_array[0].virtual_address) {
2449                 SE_DEBUG(DBG_LVL_8,
2450                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2451                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2452         } else
2453                 shost_printk(KERN_WARNING, phba->shost,
2454                              "No Virtual address\n");
2455
2456         pasync_ctx->async_header.va_base =
2457                         mem_descr->mem_array[0].virtual_address;
2458
2459         pasync_ctx->async_header.pa_base.u.a64.address =
2460                         mem_descr->mem_array[0].bus_address.u.a64.address;
2461
2462         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2463         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2464         if (mem_descr->mem_array[0].virtual_address) {
2465                 SE_DEBUG(DBG_LVL_8,
2466                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2467                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2468         } else
2469                 shost_printk(KERN_WARNING, phba->shost,
2470                             "No Virtual address\n");
2471         pasync_ctx->async_header.ring_base =
2472                         mem_descr->mem_array[0].virtual_address;
2473
2474         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2475         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2476         if (mem_descr->mem_array[0].virtual_address) {
2477                 SE_DEBUG(DBG_LVL_8,
2478                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2479                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2480         } else
2481                 shost_printk(KERN_WARNING, phba->shost,
2482                             "No Virtual address\n");
2483
2484         pasync_ctx->async_header.handle_base =
2485                         mem_descr->mem_array[0].virtual_address;
2486         pasync_ctx->async_header.writables = 0;
2487         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2488
2489         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2490         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2491         if (mem_descr->mem_array[0].virtual_address) {
2492                 SE_DEBUG(DBG_LVL_8,
2493                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2494                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2495         } else
2496                 shost_printk(KERN_WARNING, phba->shost,
2497                             "No Virtual address\n");
2498         pasync_ctx->async_data.va_base =
2499                         mem_descr->mem_array[0].virtual_address;
2500         pasync_ctx->async_data.pa_base.u.a64.address =
2501                         mem_descr->mem_array[0].bus_address.u.a64.address;
2502
2503         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2504         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2505         if (mem_descr->mem_array[0].virtual_address) {
2506                 SE_DEBUG(DBG_LVL_8,
2507                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2508                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2509         } else
2510                 shost_printk(KERN_WARNING, phba->shost,
2511                              "No Virtual address\n");
2512
2513         pasync_ctx->async_data.ring_base =
2514                         mem_descr->mem_array[0].virtual_address;
2515
2516         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2517         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2518         if (!mem_descr->mem_array[0].virtual_address)
2519                 shost_printk(KERN_WARNING, phba->shost,
2520                             "No Virtual address\n");
2521
2522         pasync_ctx->async_data.handle_base =
2523                         mem_descr->mem_array[0].virtual_address;
2524         pasync_ctx->async_data.writables = 0;
2525         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2526
2527         pasync_header_h =
2528                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2529         pasync_data_h =
2530                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2531
2532         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2533                 pasync_header_h->cri = -1;
2534                 pasync_header_h->index = (char)index;
2535                 INIT_LIST_HEAD(&pasync_header_h->link);
2536                 pasync_header_h->pbuffer =
2537                         (void *)((unsigned long)
2538                         (pasync_ctx->async_header.va_base) +
2539                         (p->defpdu_hdr_sz * index));
2540
2541                 pasync_header_h->pa.u.a64.address =
2542                         pasync_ctx->async_header.pa_base.u.a64.address +
2543                         (p->defpdu_hdr_sz * index);
2544
2545                 list_add_tail(&pasync_header_h->link,
2546                                 &pasync_ctx->async_header.free_list);
2547                 pasync_header_h++;
2548                 pasync_ctx->async_header.free_entries++;
2549                 pasync_ctx->async_header.writables++;
2550
2551                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2552                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2553                                header_busy_list);
2554                 pasync_data_h->cri = -1;
2555                 pasync_data_h->index = (char)index;
2556                 INIT_LIST_HEAD(&pasync_data_h->link);
2557                 pasync_data_h->pbuffer =
2558                         (void *)((unsigned long)
2559                         (pasync_ctx->async_data.va_base) +
2560                         (p->defpdu_data_sz * index));
2561
2562                 pasync_data_h->pa.u.a64.address =
2563                     pasync_ctx->async_data.pa_base.u.a64.address +
2564                     (p->defpdu_data_sz * index);
2565
2566                 list_add_tail(&pasync_data_h->link,
2567                               &pasync_ctx->async_data.free_list);
2568                 pasync_data_h++;
2569                 pasync_ctx->async_data.free_entries++;
2570                 pasync_ctx->async_data.writables++;
2571
2572                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2573         }
2574
2575         pasync_ctx->async_header.host_write_ptr = 0;
2576         pasync_ctx->async_header.ep_read_ptr = -1;
2577         pasync_ctx->async_data.host_write_ptr = 0;
2578         pasync_ctx->async_data.ep_read_ptr = -1;
2579 }
2580
2581 static int
2582 be_sgl_create_contiguous(void *virtual_address,
2583                          u64 physical_address, u32 length,
2584                          struct be_dma_mem *sgl)
2585 {
2586         WARN_ON(!virtual_address);
2587         WARN_ON(!physical_address);
2588         WARN_ON(!length > 0);
2589         WARN_ON(!sgl);
2590
2591         sgl->va = virtual_address;
2592         sgl->dma = (unsigned long)physical_address;
2593         sgl->size = length;
2594
2595         return 0;
2596 }
2597
2598 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2599 {
2600         memset(sgl, 0, sizeof(*sgl));
2601 }
2602
2603 static void
2604 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2605                      struct mem_array *pmem, struct be_dma_mem *sgl)
2606 {
2607         if (sgl->va)
2608                 be_sgl_destroy_contiguous(sgl);
2609
2610         be_sgl_create_contiguous(pmem->virtual_address,
2611                                  pmem->bus_address.u.a64.address,
2612                                  pmem->size, sgl);
2613 }
2614
2615 static void
2616 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2617                            struct mem_array *pmem, struct be_dma_mem *sgl)
2618 {
2619         if (sgl->va)
2620                 be_sgl_destroy_contiguous(sgl);
2621
2622         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2623                                  pmem->bus_address.u.a64.address,
2624                                  pmem->size, sgl);
2625 }
2626
2627 static int be_fill_queue(struct be_queue_info *q,
2628                 u16 len, u16 entry_size, void *vaddress)
2629 {
2630         struct be_dma_mem *mem = &q->dma_mem;
2631
2632         memset(q, 0, sizeof(*q));
2633         q->len = len;
2634         q->entry_size = entry_size;
2635         mem->size = len * entry_size;
2636         mem->va = vaddress;
2637         if (!mem->va)
2638                 return -ENOMEM;
2639         memset(mem->va, 0, mem->size);
2640         return 0;
2641 }
2642
2643 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2644                              struct hwi_context_memory *phwi_context)
2645 {
2646         unsigned int i, num_eq_pages;
2647         int ret, eq_for_mcc;
2648         struct be_queue_info *eq;
2649         struct be_dma_mem *mem;
2650         void *eq_vaddress;
2651         dma_addr_t paddr;
2652
2653         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2654                                       sizeof(struct be_eq_entry));
2655
2656         if (phba->msix_enabled)
2657                 eq_for_mcc = 1;
2658         else
2659                 eq_for_mcc = 0;
2660         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2661                 eq = &phwi_context->be_eq[i].q;
2662                 mem = &eq->dma_mem;
2663                 phwi_context->be_eq[i].phba = phba;
2664                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2665                                                      num_eq_pages * PAGE_SIZE,
2666                                                      &paddr);
2667                 if (!eq_vaddress)
2668                         goto create_eq_error;
2669
2670                 mem->va = eq_vaddress;
2671                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2672                                     sizeof(struct be_eq_entry), eq_vaddress);
2673                 if (ret) {
2674                         shost_printk(KERN_ERR, phba->shost,
2675                                      "be_fill_queue Failed for EQ\n");
2676                         goto create_eq_error;
2677                 }
2678
2679                 mem->dma = paddr;
2680                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2681                                             phwi_context->cur_eqd);
2682                 if (ret) {
2683                         shost_printk(KERN_ERR, phba->shost,
2684                                      "beiscsi_cmd_eq_create"
2685                                      "Failedfor EQ\n");
2686                         goto create_eq_error;
2687                 }
2688                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2689         }
2690         return 0;
2691 create_eq_error:
2692         for (i = 0; i < (phba->num_cpus + 1); i++) {
2693                 eq = &phwi_context->be_eq[i].q;
2694                 mem = &eq->dma_mem;
2695                 if (mem->va)
2696                         pci_free_consistent(phba->pcidev, num_eq_pages
2697                                             * PAGE_SIZE,
2698                                             mem->va, mem->dma);
2699         }
2700         return ret;
2701 }
2702
2703 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2704                              struct hwi_context_memory *phwi_context)
2705 {
2706         unsigned int i, num_cq_pages;
2707         int ret;
2708         struct be_queue_info *cq, *eq;
2709         struct be_dma_mem *mem;
2710         struct be_eq_obj *pbe_eq;
2711         void *cq_vaddress;
2712         dma_addr_t paddr;
2713
2714         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2715                                       sizeof(struct sol_cqe));
2716
2717         for (i = 0; i < phba->num_cpus; i++) {
2718                 cq = &phwi_context->be_cq[i];
2719                 eq = &phwi_context->be_eq[i].q;
2720                 pbe_eq = &phwi_context->be_eq[i];
2721                 pbe_eq->cq = cq;
2722                 pbe_eq->phba = phba;
2723                 mem = &cq->dma_mem;
2724                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2725                                                      num_cq_pages * PAGE_SIZE,
2726                                                      &paddr);
2727                 if (!cq_vaddress)
2728                         goto create_cq_error;
2729                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2730                                     sizeof(struct sol_cqe), cq_vaddress);
2731                 if (ret) {
2732                         shost_printk(KERN_ERR, phba->shost,
2733                                      "be_fill_queue Failed for ISCSI CQ\n");
2734                         goto create_cq_error;
2735                 }
2736
2737                 mem->dma = paddr;
2738                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2739                                             false, 0);
2740                 if (ret) {
2741                         shost_printk(KERN_ERR, phba->shost,
2742                                      "beiscsi_cmd_eq_create"
2743                                      "Failed for ISCSI CQ\n");
2744                         goto create_cq_error;
2745                 }
2746                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2747                                                  cq->id, eq->id);
2748                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2749         }
2750         return 0;
2751
2752 create_cq_error:
2753         for (i = 0; i < phba->num_cpus; i++) {
2754                 cq = &phwi_context->be_cq[i];
2755                 mem = &cq->dma_mem;
2756                 if (mem->va)
2757                         pci_free_consistent(phba->pcidev, num_cq_pages
2758                                             * PAGE_SIZE,
2759                                             mem->va, mem->dma);
2760         }
2761         return ret;
2762
2763 }
2764
2765 static int
2766 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2767                        struct hwi_context_memory *phwi_context,
2768                        struct hwi_controller *phwi_ctrlr,
2769                        unsigned int def_pdu_ring_sz)
2770 {
2771         unsigned int idx;
2772         int ret;
2773         struct be_queue_info *dq, *cq;
2774         struct be_dma_mem *mem;
2775         struct be_mem_descriptor *mem_descr;
2776         void *dq_vaddress;
2777
2778         idx = 0;
2779         dq = &phwi_context->be_def_hdrq;
2780         cq = &phwi_context->be_cq[0];
2781         mem = &dq->dma_mem;
2782         mem_descr = phba->init_mem;
2783         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2784         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2785         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2786                             sizeof(struct phys_addr),
2787                             sizeof(struct phys_addr), dq_vaddress);
2788         if (ret) {
2789                 shost_printk(KERN_ERR, phba->shost,
2790                              "be_fill_queue Failed for DEF PDU HDR\n");
2791                 return ret;
2792         }
2793         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2794                                   bus_address.u.a64.address;
2795         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2796                                               def_pdu_ring_sz,
2797                                               phba->params.defpdu_hdr_sz);
2798         if (ret) {
2799                 shost_printk(KERN_ERR, phba->shost,
2800                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2801                 return ret;
2802         }
2803         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2804         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2805                  phwi_context->be_def_hdrq.id);
2806         hwi_post_async_buffers(phba, 1);
2807         return 0;
2808 }
2809
2810 static int
2811 beiscsi_create_def_data(struct beiscsi_hba *phba,
2812                         struct hwi_context_memory *phwi_context,
2813                         struct hwi_controller *phwi_ctrlr,
2814                         unsigned int def_pdu_ring_sz)
2815 {
2816         unsigned int idx;
2817         int ret;
2818         struct be_queue_info *dataq, *cq;
2819         struct be_dma_mem *mem;
2820         struct be_mem_descriptor *mem_descr;
2821         void *dq_vaddress;
2822
2823         idx = 0;
2824         dataq = &phwi_context->be_def_dataq;
2825         cq = &phwi_context->be_cq[0];
2826         mem = &dataq->dma_mem;
2827         mem_descr = phba->init_mem;
2828         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2829         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2830         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2831                             sizeof(struct phys_addr),
2832                             sizeof(struct phys_addr), dq_vaddress);
2833         if (ret) {
2834                 shost_printk(KERN_ERR, phba->shost,
2835                              "be_fill_queue Failed for DEF PDU DATA\n");
2836                 return ret;
2837         }
2838         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2839                                   bus_address.u.a64.address;
2840         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2841                                               def_pdu_ring_sz,
2842                                               phba->params.defpdu_data_sz);
2843         if (ret) {
2844                 shost_printk(KERN_ERR, phba->shost,
2845                              "be_cmd_create_default_pdu_queue Failed"
2846                              " for DEF PDU DATA\n");
2847                 return ret;
2848         }
2849         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2850         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2851                  phwi_context->be_def_dataq.id);
2852         hwi_post_async_buffers(phba, 0);
2853         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
2854         return 0;
2855 }
2856
2857 static int
2858 beiscsi_post_pages(struct beiscsi_hba *phba)
2859 {
2860         struct be_mem_descriptor *mem_descr;
2861         struct mem_array *pm_arr;
2862         unsigned int page_offset, i;
2863         struct be_dma_mem sgl;
2864         int status;
2865
2866         mem_descr = phba->init_mem;
2867         mem_descr += HWI_MEM_SGE;
2868         pm_arr = mem_descr->mem_array;
2869
2870         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2871                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2872         for (i = 0; i < mem_descr->num_elements; i++) {
2873                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2874                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2875                                                 page_offset,
2876                                                 (pm_arr->size / PAGE_SIZE));
2877                 page_offset += pm_arr->size / PAGE_SIZE;
2878                 if (status != 0) {
2879                         shost_printk(KERN_ERR, phba->shost,
2880                                      "post sgl failed.\n");
2881                         return status;
2882                 }
2883                 pm_arr++;
2884         }
2885         SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
2886         return 0;
2887 }
2888
2889 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2890 {
2891         struct be_dma_mem *mem = &q->dma_mem;
2892         if (mem->va)
2893                 pci_free_consistent(phba->pcidev, mem->size,
2894                         mem->va, mem->dma);
2895 }
2896
2897 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2898                 u16 len, u16 entry_size)
2899 {
2900         struct be_dma_mem *mem = &q->dma_mem;
2901
2902         memset(q, 0, sizeof(*q));
2903         q->len = len;
2904         q->entry_size = entry_size;
2905         mem->size = len * entry_size;
2906         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2907         if (!mem->va)
2908                 return -ENOMEM;
2909         memset(mem->va, 0, mem->size);
2910         return 0;
2911 }
2912
2913 static int
2914 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2915                          struct hwi_context_memory *phwi_context,
2916                          struct hwi_controller *phwi_ctrlr)
2917 {
2918         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2919         u64 pa_addr_lo;
2920         unsigned int idx, num, i;
2921         struct mem_array *pwrb_arr;
2922         void *wrb_vaddr;
2923         struct be_dma_mem sgl;
2924         struct be_mem_descriptor *mem_descr;
2925         int status;
2926
2927         idx = 0;
2928         mem_descr = phba->init_mem;
2929         mem_descr += HWI_MEM_WRB;
2930         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2931                            GFP_KERNEL);
2932         if (!pwrb_arr) {
2933                 shost_printk(KERN_ERR, phba->shost,
2934                              "Memory alloc failed in create wrb ring.\n");
2935                 return -ENOMEM;
2936         }
2937         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2938         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2939         num_wrb_rings = mem_descr->mem_array[idx].size /
2940                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2941
2942         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2943                 if (num_wrb_rings) {
2944                         pwrb_arr[num].virtual_address = wrb_vaddr;
2945                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2946                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2947                                             sizeof(struct iscsi_wrb);
2948                         wrb_vaddr += pwrb_arr[num].size;
2949                         pa_addr_lo += pwrb_arr[num].size;
2950                         num_wrb_rings--;
2951                 } else {
2952                         idx++;
2953                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2954                         pa_addr_lo = mem_descr->mem_array[idx].\
2955                                         bus_address.u.a64.address;
2956                         num_wrb_rings = mem_descr->mem_array[idx].size /
2957                                         (phba->params.wrbs_per_cxn *
2958                                         sizeof(struct iscsi_wrb));
2959                         pwrb_arr[num].virtual_address = wrb_vaddr;
2960                         pwrb_arr[num].bus_address.u.a64.address\
2961                                                 = pa_addr_lo;
2962                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2963                                                  sizeof(struct iscsi_wrb);
2964                         wrb_vaddr += pwrb_arr[num].size;
2965                         pa_addr_lo   += pwrb_arr[num].size;
2966                         num_wrb_rings--;
2967                 }
2968         }
2969         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2970                 wrb_mem_index = 0;
2971                 offset = 0;
2972                 size = 0;
2973
2974                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2975                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2976                                             &phwi_context->be_wrbq[i]);
2977                 if (status != 0) {
2978                         shost_printk(KERN_ERR, phba->shost,
2979                                      "wrbq create failed.");
2980                         kfree(pwrb_arr);
2981                         return status;
2982                 }
2983                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2984                                                                    id;
2985         }
2986         kfree(pwrb_arr);
2987         return 0;
2988 }
2989
2990 static void free_wrb_handles(struct beiscsi_hba *phba)
2991 {
2992         unsigned int index;
2993         struct hwi_controller *phwi_ctrlr;
2994         struct hwi_wrb_context *pwrb_context;
2995
2996         phwi_ctrlr = phba->phwi_ctrlr;
2997         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2998                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2999                 kfree(pwrb_context->pwrb_handle_base);
3000                 kfree(pwrb_context->pwrb_handle_basestd);
3001         }
3002 }
3003
3004 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3005 {
3006         struct be_queue_info *q;
3007         struct be_ctrl_info *ctrl = &phba->ctrl;
3008
3009         q = &phba->ctrl.mcc_obj.q;
3010         if (q->created)
3011                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3012         be_queue_free(phba, q);
3013
3014         q = &phba->ctrl.mcc_obj.cq;
3015         if (q->created)
3016                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3017         be_queue_free(phba, q);
3018 }
3019
3020 static void hwi_cleanup(struct beiscsi_hba *phba)
3021 {
3022         struct be_queue_info *q;
3023         struct be_ctrl_info *ctrl = &phba->ctrl;
3024         struct hwi_controller *phwi_ctrlr;
3025         struct hwi_context_memory *phwi_context;
3026         int i, eq_num;
3027
3028         phwi_ctrlr = phba->phwi_ctrlr;
3029         phwi_context = phwi_ctrlr->phwi_ctxt;
3030         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3031                 q = &phwi_context->be_wrbq[i];
3032                 if (q->created)
3033                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3034         }
3035         free_wrb_handles(phba);
3036
3037         q = &phwi_context->be_def_hdrq;
3038         if (q->created)
3039                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3040
3041         q = &phwi_context->be_def_dataq;
3042         if (q->created)
3043                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3044
3045         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3046
3047         for (i = 0; i < (phba->num_cpus); i++) {
3048                 q = &phwi_context->be_cq[i];
3049                 if (q->created)
3050                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3051         }
3052         if (phba->msix_enabled)
3053                 eq_num = 1;
3054         else
3055                 eq_num = 0;
3056         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3057                 q = &phwi_context->be_eq[i].q;
3058                 if (q->created)
3059                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3060         }
3061         be_mcc_queues_destroy(phba);
3062 }
3063
3064 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3065                                 struct hwi_context_memory *phwi_context)
3066 {
3067         struct be_queue_info *q, *cq;
3068         struct be_ctrl_info *ctrl = &phba->ctrl;
3069
3070         /* Alloc MCC compl queue */
3071         cq = &phba->ctrl.mcc_obj.cq;
3072         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3073                         sizeof(struct be_mcc_compl)))
3074                 goto err;
3075         /* Ask BE to create MCC compl queue; */
3076         if (phba->msix_enabled) {
3077                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3078                                          [phba->num_cpus].q, false, true, 0))
3079                 goto mcc_cq_free;
3080         } else {
3081                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3082                                           false, true, 0))
3083                 goto mcc_cq_free;
3084         }
3085
3086         /* Alloc MCC queue */
3087         q = &phba->ctrl.mcc_obj.q;
3088         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3089                 goto mcc_cq_destroy;
3090
3091         /* Ask BE to create MCC queue */
3092         if (beiscsi_cmd_mccq_create(phba, q, cq))
3093                 goto mcc_q_free;
3094
3095         return 0;
3096
3097 mcc_q_free:
3098         be_queue_free(phba, q);
3099 mcc_cq_destroy:
3100         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3101 mcc_cq_free:
3102         be_queue_free(phba, cq);
3103 err:
3104         return -ENOMEM;
3105 }
3106
3107 static int find_num_cpus(void)
3108 {
3109         int  num_cpus = 0;
3110
3111         num_cpus = num_online_cpus();
3112         if (num_cpus >= MAX_CPUS)
3113                 num_cpus = MAX_CPUS - 1;
3114
3115         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
3116         return num_cpus;
3117 }
3118
3119 static int hwi_init_port(struct beiscsi_hba *phba)
3120 {
3121         struct hwi_controller *phwi_ctrlr;
3122         struct hwi_context_memory *phwi_context;
3123         unsigned int def_pdu_ring_sz;
3124         struct be_ctrl_info *ctrl = &phba->ctrl;
3125         int status;
3126
3127         def_pdu_ring_sz =
3128                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3129         phwi_ctrlr = phba->phwi_ctrlr;
3130         phwi_context = phwi_ctrlr->phwi_ctxt;
3131         phwi_context->max_eqd = 0;
3132         phwi_context->min_eqd = 0;
3133         phwi_context->cur_eqd = 64;
3134         be_cmd_fw_initialize(&phba->ctrl);
3135
3136         status = beiscsi_create_eqs(phba, phwi_context);
3137         if (status != 0) {
3138                 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
3139                 goto error;
3140         }
3141
3142         status = be_mcc_queues_create(phba, phwi_context);
3143         if (status != 0)
3144                 goto error;
3145
3146         status = mgmt_check_supported_fw(ctrl, phba);
3147         if (status != 0) {
3148                 shost_printk(KERN_ERR, phba->shost,
3149                              "Unsupported fw version\n");
3150                 goto error;
3151         }
3152
3153         status = beiscsi_create_cqs(phba, phwi_context);
3154         if (status != 0) {
3155                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3156                 goto error;
3157         }
3158
3159         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3160                                         def_pdu_ring_sz);
3161         if (status != 0) {
3162                 shost_printk(KERN_ERR, phba->shost,
3163                              "Default Header not created\n");
3164                 goto error;
3165         }
3166
3167         status = beiscsi_create_def_data(phba, phwi_context,
3168                                          phwi_ctrlr, def_pdu_ring_sz);
3169         if (status != 0) {
3170                 shost_printk(KERN_ERR, phba->shost,
3171                              "Default Data not created\n");
3172                 goto error;
3173         }
3174
3175         status = beiscsi_post_pages(phba);
3176         if (status != 0) {
3177                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3178                 goto error;
3179         }
3180
3181         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3182         if (status != 0) {
3183                 shost_printk(KERN_ERR, phba->shost,
3184                              "WRB Rings not created\n");
3185                 goto error;
3186         }
3187
3188         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3189         return 0;
3190
3191 error:
3192         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3193         hwi_cleanup(phba);
3194         return -ENOMEM;
3195 }
3196
3197 static int hwi_init_controller(struct beiscsi_hba *phba)
3198 {
3199         struct hwi_controller *phwi_ctrlr;
3200
3201         phwi_ctrlr = phba->phwi_ctrlr;
3202         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3203                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3204                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3205                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
3206                          phwi_ctrlr->phwi_ctxt);
3207         } else {
3208                 shost_printk(KERN_ERR, phba->shost,
3209                              "HWI_MEM_ADDN_CONTEXT is more than one element."
3210                              "Failing to load\n");
3211                 return -ENOMEM;
3212         }
3213
3214         iscsi_init_global_templates(phba);
3215         beiscsi_init_wrb_handle(phba);
3216         hwi_init_async_pdu_ctx(phba);
3217         if (hwi_init_port(phba) != 0) {
3218                 shost_printk(KERN_ERR, phba->shost,
3219                              "hwi_init_controller failed\n");
3220                 return -ENOMEM;
3221         }
3222         return 0;
3223 }
3224
3225 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3226 {
3227         struct be_mem_descriptor *mem_descr;
3228         int i, j;
3229
3230         mem_descr = phba->init_mem;
3231         i = 0;
3232         j = 0;
3233         for (i = 0; i < SE_MEM_MAX; i++) {
3234                 for (j = mem_descr->num_elements; j > 0; j--) {
3235                         pci_free_consistent(phba->pcidev,
3236                           mem_descr->mem_array[j - 1].size,
3237                           mem_descr->mem_array[j - 1].virtual_address,
3238                           (unsigned long)mem_descr->mem_array[j - 1].
3239                           bus_address.u.a64.address);
3240                 }
3241                 kfree(mem_descr->mem_array);
3242                 mem_descr++;
3243         }
3244         kfree(phba->init_mem);
3245         kfree(phba->phwi_ctrlr);
3246 }
3247
3248 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3249 {
3250         int ret = -ENOMEM;
3251
3252         ret = beiscsi_get_memory(phba);
3253         if (ret < 0) {
3254                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3255                              "Failed in beiscsi_alloc_memory\n");
3256                 return ret;
3257         }
3258
3259         ret = hwi_init_controller(phba);
3260         if (ret)
3261                 goto free_init;
3262         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3263         return 0;
3264
3265 free_init:
3266         beiscsi_free_mem(phba);
3267         return -ENOMEM;
3268 }
3269
3270 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3271 {
3272         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3273         struct sgl_handle *psgl_handle;
3274         struct iscsi_sge *pfrag;
3275         unsigned int arr_index, i, idx;
3276
3277         phba->io_sgl_hndl_avbl = 0;
3278         phba->eh_sgl_hndl_avbl = 0;
3279
3280         mem_descr_sglh = phba->init_mem;
3281         mem_descr_sglh += HWI_MEM_SGLH;
3282         if (1 == mem_descr_sglh->num_elements) {
3283                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3284                                                  phba->params.ios_per_ctrl,
3285                                                  GFP_KERNEL);
3286                 if (!phba->io_sgl_hndl_base) {
3287                         shost_printk(KERN_ERR, phba->shost,
3288                                      "Mem Alloc Failed. Failing to load\n");
3289                         return -ENOMEM;
3290                 }
3291                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3292                                                  (phba->params.icds_per_ctrl -
3293                                                  phba->params.ios_per_ctrl),
3294                                                  GFP_KERNEL);
3295                 if (!phba->eh_sgl_hndl_base) {
3296                         kfree(phba->io_sgl_hndl_base);
3297                         shost_printk(KERN_ERR, phba->shost,
3298                                      "Mem Alloc Failed. Failing to load\n");
3299                         return -ENOMEM;
3300                 }
3301         } else {
3302                 shost_printk(KERN_ERR, phba->shost,
3303                              "HWI_MEM_SGLH is more than one element."
3304                              "Failing to load\n");
3305                 return -ENOMEM;
3306         }
3307
3308         arr_index = 0;
3309         idx = 0;
3310         while (idx < mem_descr_sglh->num_elements) {
3311                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3312
3313                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3314                       sizeof(struct sgl_handle)); i++) {
3315                         if (arr_index < phba->params.ios_per_ctrl) {
3316                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3317                                 phba->io_sgl_hndl_avbl++;
3318                                 arr_index++;
3319                         } else {
3320                                 phba->eh_sgl_hndl_base[arr_index -
3321                                         phba->params.ios_per_ctrl] =
3322                                                                 psgl_handle;
3323                                 arr_index++;
3324                                 phba->eh_sgl_hndl_avbl++;
3325                         }
3326                         psgl_handle++;
3327                 }
3328                 idx++;
3329         }
3330         SE_DEBUG(DBG_LVL_8,
3331                  "phba->io_sgl_hndl_avbl=%d"
3332                  "phba->eh_sgl_hndl_avbl=%d\n",
3333                  phba->io_sgl_hndl_avbl,
3334                  phba->eh_sgl_hndl_avbl);
3335         mem_descr_sg = phba->init_mem;
3336         mem_descr_sg += HWI_MEM_SGE;
3337         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
3338                  mem_descr_sg->num_elements);
3339         arr_index = 0;
3340         idx = 0;
3341         while (idx < mem_descr_sg->num_elements) {
3342                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3343
3344                 for (i = 0;
3345                      i < (mem_descr_sg->mem_array[idx].size) /
3346                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3347                      i++) {
3348                         if (arr_index < phba->params.ios_per_ctrl)
3349                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3350                         else
3351                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3352                                                 phba->params.ios_per_ctrl];
3353                         psgl_handle->pfrag = pfrag;
3354                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3355                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3356                         pfrag += phba->params.num_sge_per_io;
3357                         psgl_handle->sgl_index =
3358                                 phba->fw_config.iscsi_icd_start + arr_index++;
3359                 }
3360                 idx++;
3361         }
3362         phba->io_sgl_free_index = 0;
3363         phba->io_sgl_alloc_index = 0;
3364         phba->eh_sgl_free_index = 0;
3365         phba->eh_sgl_alloc_index = 0;
3366         return 0;
3367 }
3368
3369 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3370 {
3371         int i, new_cid;
3372
3373         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3374                                   GFP_KERNEL);
3375         if (!phba->cid_array) {
3376                 shost_printk(KERN_ERR, phba->shost,
3377                              "Failed to allocate memory in "
3378                              "hba_setup_cid_tbls\n");
3379                 return -ENOMEM;
3380         }
3381         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3382                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3383         if (!phba->ep_array) {
3384                 shost_printk(KERN_ERR, phba->shost,
3385                              "Failed to allocate memory in "
3386                              "hba_setup_cid_tbls\n");
3387                 kfree(phba->cid_array);
3388                 return -ENOMEM;
3389         }
3390         new_cid = phba->fw_config.iscsi_cid_start;
3391         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3392                 phba->cid_array[i] = new_cid;
3393                 new_cid += 2;
3394         }
3395         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3396         return 0;
3397 }
3398
3399 static void hwi_enable_intr(struct beiscsi_hba *phba)
3400 {
3401         struct be_ctrl_info *ctrl = &phba->ctrl;
3402         struct hwi_controller *phwi_ctrlr;
3403         struct hwi_context_memory *phwi_context;
3404         struct be_queue_info *eq;
3405         u8 __iomem *addr;
3406         u32 reg, i;
3407         u32 enabled;
3408
3409         phwi_ctrlr = phba->phwi_ctrlr;
3410         phwi_context = phwi_ctrlr->phwi_ctxt;
3411
3412         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3413                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3414         reg = ioread32(addr);
3415
3416         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3417         if (!enabled) {
3418                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3419                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
3420                 iowrite32(reg, addr);
3421         }
3422
3423         if (!phba->msix_enabled) {
3424                 eq = &phwi_context->be_eq[0].q;
3425                 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3426                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3427         } else {
3428                 for (i = 0; i <= phba->num_cpus; i++) {
3429                         eq = &phwi_context->be_eq[i].q;
3430                         SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3431                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3432                 }
3433         }
3434 }
3435
3436 static void hwi_disable_intr(struct beiscsi_hba *phba)
3437 {
3438         struct be_ctrl_info *ctrl = &phba->ctrl;
3439
3440         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3441         u32 reg = ioread32(addr);
3442
3443         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3444         if (enabled) {
3445                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3446                 iowrite32(reg, addr);
3447         } else
3448                 shost_printk(KERN_WARNING, phba->shost,
3449                              "In hwi_disable_intr, Already Disabled\n");
3450 }
3451
3452 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3453 {
3454         struct be_cmd_resp_get_boot_target *boot_resp;
3455         struct be_cmd_resp_get_session *session_resp;
3456         struct be_mcc_wrb *wrb;
3457         struct be_dma_mem nonemb_cmd;
3458         unsigned int tag, wrb_num;
3459         unsigned short status, extd_status;
3460         struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3461         int ret = -ENOMEM;
3462
3463         tag = beiscsi_get_boot_target(phba);
3464         if (!tag) {
3465                 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
3466                 return -EAGAIN;
3467         } else
3468                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3469                                          phba->ctrl.mcc_numtag[tag]);
3470
3471         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3472         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3473         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3474         if (status || extd_status) {
3475                 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
3476                                     " status = %d extd_status = %d\n",
3477                                     status, extd_status);
3478                 free_mcc_tag(&phba->ctrl, tag);
3479                 return -EBUSY;
3480         }
3481         wrb = queue_get_wrb(mccq, wrb_num);
3482         free_mcc_tag(&phba->ctrl, tag);
3483         boot_resp = embedded_payload(wrb);
3484
3485         if (boot_resp->boot_session_handle < 0) {
3486                 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
3487                 return -ENXIO;
3488         }
3489
3490         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3491                                 sizeof(*session_resp),
3492                                 &nonemb_cmd.dma);
3493         if (nonemb_cmd.va == NULL) {
3494                 SE_DEBUG(DBG_LVL_1,
3495                          "Failed to allocate memory for"
3496                          "beiscsi_get_session_info\n");
3497                 return -ENOMEM;
3498         }
3499
3500         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3501         tag = beiscsi_get_session_info(phba,
3502                 boot_resp->boot_session_handle, &nonemb_cmd);
3503         if (!tag) {
3504                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3505                         " Failed\n");
3506                 goto boot_freemem;
3507         } else
3508                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3509                                          phba->ctrl.mcc_numtag[tag]);
3510
3511         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3512         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3513         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3514         if (status || extd_status) {
3515                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3516                                     " status = %d extd_status = %d\n",
3517                                     status, extd_status);
3518                 free_mcc_tag(&phba->ctrl, tag);
3519                 goto boot_freemem;
3520         }
3521         wrb = queue_get_wrb(mccq, wrb_num);
3522         free_mcc_tag(&phba->ctrl, tag);
3523         session_resp = nonemb_cmd.va ;
3524
3525         memcpy(&phba->boot_sess, &session_resp->session_info,
3526                sizeof(struct mgmt_session_info));
3527         ret = 0;
3528
3529 boot_freemem:
3530         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3531                     nonemb_cmd.va, nonemb_cmd.dma);
3532         return ret;
3533 }
3534
3535 static void beiscsi_boot_release(void *data)
3536 {
3537         struct beiscsi_hba *phba = data;
3538
3539         scsi_host_put(phba->shost);
3540 }
3541
3542 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3543 {
3544         struct iscsi_boot_kobj *boot_kobj;
3545
3546         /* get boot info using mgmt cmd */
3547         if (beiscsi_get_boot_info(phba))
3548                 /* Try to see if we can carry on without this */
3549                 return 0;
3550
3551         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3552         if (!phba->boot_kset)
3553                 return -ENOMEM;
3554
3555         /* get a ref because the show function will ref the phba */
3556         if (!scsi_host_get(phba->shost))
3557                 goto free_kset;
3558         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3559                                              beiscsi_show_boot_tgt_info,
3560                                              beiscsi_tgt_get_attr_visibility,
3561                                              beiscsi_boot_release);
3562         if (!boot_kobj)
3563                 goto put_shost;
3564
3565         if (!scsi_host_get(phba->shost))
3566                 goto free_kset;
3567         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3568                                                 beiscsi_show_boot_ini_info,
3569                                                 beiscsi_ini_get_attr_visibility,
3570                                                 beiscsi_boot_release);
3571         if (!boot_kobj)
3572                 goto put_shost;
3573
3574         if (!scsi_host_get(phba->shost))
3575                 goto free_kset;
3576         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3577                                                beiscsi_show_boot_eth_info,
3578                                                beiscsi_eth_get_attr_visibility,
3579                                                beiscsi_boot_release);
3580         if (!boot_kobj)
3581                 goto put_shost;
3582         return 0;
3583
3584 put_shost:
3585         scsi_host_put(phba->shost);
3586 free_kset:
3587         iscsi_boot_destroy_kset(phba->boot_kset);
3588         return -ENOMEM;
3589 }
3590
3591 static int beiscsi_init_port(struct beiscsi_hba *phba)
3592 {
3593         int ret;
3594
3595         ret = beiscsi_init_controller(phba);
3596         if (ret < 0) {
3597                 shost_printk(KERN_ERR, phba->shost,
3598                              "beiscsi_dev_probe - Failed in"
3599                              "beiscsi_init_controller\n");
3600                 return ret;
3601         }
3602         ret = beiscsi_init_sgl_handle(phba);
3603         if (ret < 0) {
3604                 shost_printk(KERN_ERR, phba->shost,
3605                              "beiscsi_dev_probe - Failed in"
3606                              "beiscsi_init_sgl_handle\n");
3607                 goto do_cleanup_ctrlr;
3608         }
3609
3610         if (hba_setup_cid_tbls(phba)) {
3611                 shost_printk(KERN_ERR, phba->shost,
3612                              "Failed in hba_setup_cid_tbls\n");
3613                 kfree(phba->io_sgl_hndl_base);
3614                 kfree(phba->eh_sgl_hndl_base);
3615                 goto do_cleanup_ctrlr;
3616         }
3617
3618         return ret;
3619
3620 do_cleanup_ctrlr:
3621         hwi_cleanup(phba);
3622         return ret;
3623 }
3624
3625 static void hwi_purge_eq(struct beiscsi_hba *phba)
3626 {
3627         struct hwi_controller *phwi_ctrlr;
3628         struct hwi_context_memory *phwi_context;
3629         struct be_queue_info *eq;
3630         struct be_eq_entry *eqe = NULL;
3631         int i, eq_msix;
3632         unsigned int num_processed;
3633
3634         phwi_ctrlr = phba->phwi_ctrlr;
3635         phwi_context = phwi_ctrlr->phwi_ctxt;
3636         if (phba->msix_enabled)
3637                 eq_msix = 1;
3638         else
3639                 eq_msix = 0;
3640
3641         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3642                 eq = &phwi_context->be_eq[i].q;
3643                 eqe = queue_tail_node(eq);
3644                 num_processed = 0;
3645                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3646                                         & EQE_VALID_MASK) {
3647                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3648                         queue_tail_inc(eq);
3649                         eqe = queue_tail_node(eq);
3650                         num_processed++;
3651                 }
3652
3653                 if (num_processed)
3654                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3655         }
3656 }
3657
3658 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3659 {
3660         int mgmt_status;
3661
3662         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3663         if (mgmt_status)
3664                 shost_printk(KERN_WARNING, phba->shost,
3665                              "mgmt_epfw_cleanup FAILED\n");
3666
3667         hwi_purge_eq(phba);
3668         hwi_cleanup(phba);
3669         kfree(phba->io_sgl_hndl_base);
3670         kfree(phba->eh_sgl_hndl_base);
3671         kfree(phba->cid_array);
3672         kfree(phba->ep_array);
3673 }
3674
3675 void
3676 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3677                            struct beiscsi_offload_params *params)
3678 {
3679         struct wrb_handle *pwrb_handle;
3680         struct iscsi_target_context_update_wrb *pwrb = NULL;
3681         struct be_mem_descriptor *mem_descr;
3682         struct beiscsi_hba *phba = beiscsi_conn->phba;
3683         u32 doorbell = 0;
3684
3685         /*
3686          * We can always use 0 here because it is reserved by libiscsi for
3687          * login/startup related tasks.
3688          */
3689         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3690                                        phba->fw_config.iscsi_cid_start));
3691         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3692         memset(pwrb, 0, sizeof(*pwrb));
3693         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3694                       max_burst_length, pwrb, params->dw[offsetof
3695                       (struct amap_beiscsi_offload_params,
3696                       max_burst_length) / 32]);
3697         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3698                       max_send_data_segment_length, pwrb,
3699                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3700                       max_send_data_segment_length) / 32]);
3701         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3702                       first_burst_length,
3703                       pwrb,
3704                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3705                       first_burst_length) / 32]);
3706
3707         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3708                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3709                       erl) / 32] & OFFLD_PARAMS_ERL));
3710         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3711                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3712                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3713         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3714                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3715                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3716         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3717                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3718                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3719         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3720                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3721                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3722         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3723                       pwrb,
3724                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3725                       exp_statsn) / 32] + 1));
3726         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3727                       0x7);
3728         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3729                       pwrb, pwrb_handle->wrb_index);
3730         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3731                       pwrb, pwrb_handle->nxt_wrb_index);
3732         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3733                         session_state, pwrb, 0);
3734         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3735                       pwrb, 1);
3736         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3737                       pwrb, 0);
3738         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3739                       0);
3740
3741         mem_descr = phba->init_mem;
3742         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3743
3744         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3745                         pad_buffer_addr_hi, pwrb,
3746                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3747         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3748                         pad_buffer_addr_lo, pwrb,
3749                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3750
3751         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3752
3753         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3754         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3755                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3756         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3757
3758         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3759 }
3760
3761 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3762                               int *index, int *age)
3763 {
3764         *index = (int)itt;
3765         if (age)
3766                 *age = conn->session->age;
3767 }
3768
3769 /**
3770  * beiscsi_alloc_pdu - allocates pdu and related resources
3771  * @task: libiscsi task
3772  * @opcode: opcode of pdu for task
3773  *
3774  * This is called with the session lock held. It will allocate
3775  * the wrb and sgl if needed for the command. And it will prep
3776  * the pdu's itt. beiscsi_parse_pdu will later translate
3777  * the pdu itt to the libiscsi task itt.
3778  */
3779 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3780 {
3781         struct beiscsi_io_task *io_task = task->dd_data;
3782         struct iscsi_conn *conn = task->conn;
3783         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3784         struct beiscsi_hba *phba = beiscsi_conn->phba;
3785         struct hwi_wrb_context *pwrb_context;
3786         struct hwi_controller *phwi_ctrlr;
3787         itt_t itt;
3788         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3789         dma_addr_t paddr;
3790
3791         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3792                                           GFP_ATOMIC, &paddr);
3793         if (!io_task->cmd_bhs)
3794                 return -ENOMEM;
3795         io_task->bhs_pa.u.a64.address = paddr;
3796         io_task->libiscsi_itt = (itt_t)task->itt;
3797         io_task->conn = beiscsi_conn;
3798
3799         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3800         task->hdr_max = sizeof(struct be_cmd_bhs);
3801         io_task->psgl_handle = NULL;
3802         io_task->psgl_handle = NULL;
3803
3804         if (task->sc) {
3805                 spin_lock(&phba->io_sgl_lock);
3806                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3807                 spin_unlock(&phba->io_sgl_lock);
3808                 if (!io_task->psgl_handle)
3809                         goto free_hndls;
3810                 io_task->pwrb_handle = alloc_wrb_handle(phba,
3811                                         beiscsi_conn->beiscsi_conn_cid -
3812                                         phba->fw_config.iscsi_cid_start);
3813                 if (!io_task->pwrb_handle)
3814                         goto free_io_hndls;
3815         } else {
3816                 io_task->scsi_cmnd = NULL;
3817                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3818                         if (!beiscsi_conn->login_in_progress) {
3819                                 spin_lock(&phba->mgmt_sgl_lock);
3820                                 io_task->psgl_handle = (struct sgl_handle *)
3821                                                 alloc_mgmt_sgl_handle(phba);
3822                                 spin_unlock(&phba->mgmt_sgl_lock);
3823                                 if (!io_task->psgl_handle)
3824                                         goto free_hndls;
3825
3826                                 beiscsi_conn->login_in_progress = 1;
3827                                 beiscsi_conn->plogin_sgl_handle =
3828                                                         io_task->psgl_handle;
3829                                 io_task->pwrb_handle =
3830                                         alloc_wrb_handle(phba,
3831                                         beiscsi_conn->beiscsi_conn_cid -
3832                                         phba->fw_config.iscsi_cid_start);
3833                                 if (!io_task->pwrb_handle)
3834                                         goto free_io_hndls;
3835                                 beiscsi_conn->plogin_wrb_handle =
3836                                                         io_task->pwrb_handle;
3837
3838                         } else {
3839                                 io_task->psgl_handle =
3840                                                 beiscsi_conn->plogin_sgl_handle;
3841                                 io_task->pwrb_handle =
3842                                                 beiscsi_conn->plogin_wrb_handle;
3843                         }
3844                 } else {
3845                         spin_lock(&phba->mgmt_sgl_lock);
3846                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3847                         spin_unlock(&phba->mgmt_sgl_lock);
3848                         if (!io_task->psgl_handle)
3849                                 goto free_hndls;
3850                         io_task->pwrb_handle =
3851                                         alloc_wrb_handle(phba,
3852                                         beiscsi_conn->beiscsi_conn_cid -
3853                                         phba->fw_config.iscsi_cid_start);
3854                         if (!io_task->pwrb_handle)
3855                                 goto free_mgmt_hndls;
3856
3857                 }
3858         }
3859         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3860                                  wrb_index << 16) | (unsigned int)
3861                                 (io_task->psgl_handle->sgl_index));
3862         io_task->pwrb_handle->pio_handle = task;
3863
3864         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3865         return 0;
3866
3867 free_io_hndls:
3868         spin_lock(&phba->io_sgl_lock);
3869         free_io_sgl_handle(phba, io_task->psgl_handle);
3870         spin_unlock(&phba->io_sgl_lock);
3871         goto free_hndls;
3872 free_mgmt_hndls:
3873         spin_lock(&phba->mgmt_sgl_lock);
3874         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3875         spin_unlock(&phba->mgmt_sgl_lock);
3876 free_hndls:
3877         phwi_ctrlr = phba->phwi_ctrlr;
3878         pwrb_context = &phwi_ctrlr->wrb_context[
3879                         beiscsi_conn->beiscsi_conn_cid -
3880                         phba->fw_config.iscsi_cid_start];
3881         if (io_task->pwrb_handle)
3882                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3883         io_task->pwrb_handle = NULL;
3884         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3885                       io_task->bhs_pa.u.a64.address);
3886         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
3887         return -ENOMEM;
3888 }
3889
3890 static void beiscsi_cleanup_task(struct iscsi_task *task)
3891 {
3892         struct beiscsi_io_task *io_task = task->dd_data;
3893         struct iscsi_conn *conn = task->conn;
3894         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3895         struct beiscsi_hba *phba = beiscsi_conn->phba;
3896         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3897         struct hwi_wrb_context *pwrb_context;
3898         struct hwi_controller *phwi_ctrlr;
3899
3900         phwi_ctrlr = phba->phwi_ctrlr;
3901         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3902                         - phba->fw_config.iscsi_cid_start];
3903         if (io_task->pwrb_handle) {
3904                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3905                 io_task->pwrb_handle = NULL;
3906         }
3907
3908         if (io_task->cmd_bhs) {
3909                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3910                               io_task->bhs_pa.u.a64.address);
3911         }
3912
3913         if (task->sc) {
3914                 if (io_task->psgl_handle) {
3915                         spin_lock(&phba->io_sgl_lock);
3916                         free_io_sgl_handle(phba, io_task->psgl_handle);
3917                         spin_unlock(&phba->io_sgl_lock);
3918                         io_task->psgl_handle = NULL;
3919                 }
3920         } else {
3921                 if (task->hdr &&
3922                    ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN))
3923                         return;
3924                 if (io_task->psgl_handle) {
3925                         spin_lock(&phba->mgmt_sgl_lock);
3926                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3927                         spin_unlock(&phba->mgmt_sgl_lock);
3928                         io_task->psgl_handle = NULL;
3929                 }
3930         }
3931 }
3932
3933 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3934                           unsigned int num_sg, unsigned int xferlen,
3935                           unsigned int writedir)
3936 {
3937
3938         struct beiscsi_io_task *io_task = task->dd_data;
3939         struct iscsi_conn *conn = task->conn;
3940         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3941         struct beiscsi_hba *phba = beiscsi_conn->phba;
3942         struct iscsi_wrb *pwrb = NULL;
3943         unsigned int doorbell = 0;
3944
3945         pwrb = io_task->pwrb_handle->pwrb;
3946         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3947         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3948
3949         if (writedir) {
3950                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3951                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3952                               &io_task->cmd_bhs->iscsi_data_pdu,
3953                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3954                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3955                               &io_task->cmd_bhs->iscsi_data_pdu,
3956                               ISCSI_OPCODE_SCSI_DATA_OUT);
3957                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3958                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3959                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3960                               INI_WR_CMD);
3961                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3962         } else {
3963                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3964                               INI_RD_CMD);
3965                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3966         }
3967         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3968                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3969                &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3970
3971         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3972                       cpu_to_be16(*(unsigned short *)&io_task->cmd_bhs->iscsi_hdr.lun));
3973         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3974         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3975                       io_task->pwrb_handle->wrb_index);
3976         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3977                       be32_to_cpu(task->cmdsn));
3978         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3979                       io_task->psgl_handle->sgl_index);
3980
3981         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3982
3983         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3984                       io_task->pwrb_handle->nxt_wrb_index);
3985         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3986
3987         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3988         doorbell |= (io_task->pwrb_handle->wrb_index &
3989                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3990         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3991
3992         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3993         return 0;
3994 }
3995
3996 static int beiscsi_mtask(struct iscsi_task *task)
3997 {
3998         struct beiscsi_io_task *io_task = task->dd_data;
3999         struct iscsi_conn *conn = task->conn;
4000         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4001         struct beiscsi_hba *phba = beiscsi_conn->phba;
4002         struct iscsi_wrb *pwrb = NULL;
4003         unsigned int doorbell = 0;
4004         unsigned int cid;
4005
4006         cid = beiscsi_conn->beiscsi_conn_cid;
4007         pwrb = io_task->pwrb_handle->pwrb;
4008         memset(pwrb, 0, sizeof(*pwrb));
4009         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4010                       be32_to_cpu(task->cmdsn));
4011         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4012                       io_task->pwrb_handle->wrb_index);
4013         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4014                       io_task->psgl_handle->sgl_index);
4015
4016         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4017         case ISCSI_OP_LOGIN:
4018                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4019                               TGT_DM_CMD);
4020                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4021                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4022                 hwi_write_buffer(pwrb, task);
4023                 break;
4024         case ISCSI_OP_NOOP_OUT:
4025                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4026                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4027                                       TGT_DM_CMD);
4028                         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4029                                       pwrb, 0);
4030                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4031                 } else {
4032                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4033                                       INI_RD_CMD);
4034                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
4035                 }
4036                 hwi_write_buffer(pwrb, task);
4037                 break;
4038         case ISCSI_OP_TEXT:
4039                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4040                               TGT_DM_CMD);
4041                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4042                 hwi_write_buffer(pwrb, task);
4043                 break;
4044         case ISCSI_OP_SCSI_TMFUNC:
4045                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4046                               INI_TMF_CMD);
4047                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4048                 hwi_write_buffer(pwrb, task);
4049                 break;
4050         case ISCSI_OP_LOGOUT:
4051                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4052                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4053                               HWH_TYPE_LOGOUT);
4054                 hwi_write_buffer(pwrb, task);
4055                 break;
4056
4057         default:
4058                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
4059                          task->hdr->opcode & ISCSI_OPCODE_MASK);
4060                 return -EINVAL;
4061         }
4062
4063         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4064                       task->data_count);
4065         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4066                       io_task->pwrb_handle->nxt_wrb_index);
4067         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4068
4069         doorbell |= cid & DB_WRB_POST_CID_MASK;
4070         doorbell |= (io_task->pwrb_handle->wrb_index &
4071                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4072         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4073         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4074         return 0;
4075 }
4076
4077 static int beiscsi_task_xmit(struct iscsi_task *task)
4078 {
4079         struct beiscsi_io_task *io_task = task->dd_data;
4080         struct scsi_cmnd *sc = task->sc;
4081         struct scatterlist *sg;
4082         int num_sg;
4083         unsigned int  writedir = 0, xferlen = 0;
4084
4085         if (!sc)
4086                 return beiscsi_mtask(task);
4087
4088         io_task->scsi_cmnd = sc;
4089         num_sg = scsi_dma_map(sc);
4090         if (num_sg < 0) {
4091                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4092                 return num_sg;
4093         }
4094         xferlen = scsi_bufflen(sc);
4095         sg = scsi_sglist(sc);
4096         if (sc->sc_data_direction == DMA_TO_DEVICE) {
4097                 writedir = 1;
4098                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
4099                          task->imm_count);
4100         } else
4101                 writedir = 0;
4102         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4103 }
4104
4105 static void beiscsi_remove(struct pci_dev *pcidev)
4106 {
4107         struct beiscsi_hba *phba = NULL;
4108         struct hwi_controller *phwi_ctrlr;
4109         struct hwi_context_memory *phwi_context;
4110         struct be_eq_obj *pbe_eq;
4111         unsigned int i, msix_vec;
4112         u8 *real_offset = 0;
4113         u32 value = 0;
4114
4115         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4116         if (!phba) {
4117                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4118                 return;
4119         }
4120
4121         phwi_ctrlr = phba->phwi_ctrlr;
4122         phwi_context = phwi_ctrlr->phwi_ctxt;
4123         hwi_disable_intr(phba);
4124         if (phba->msix_enabled) {
4125                 for (i = 0; i <= phba->num_cpus; i++) {
4126                         msix_vec = phba->msix_entries[i].vector;
4127                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4128                 }
4129         } else
4130                 if (phba->pcidev->irq)
4131                         free_irq(phba->pcidev->irq, phba);
4132         pci_disable_msix(phba->pcidev);
4133         destroy_workqueue(phba->wq);
4134         if (blk_iopoll_enabled)
4135                 for (i = 0; i < phba->num_cpus; i++) {
4136                         pbe_eq = &phwi_context->be_eq[i];
4137                         blk_iopoll_disable(&pbe_eq->iopoll);
4138                 }
4139
4140         beiscsi_clean_port(phba);
4141         beiscsi_free_mem(phba);
4142         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4143
4144         value = readl((void *)real_offset);
4145
4146         if (value & 0x00010000) {
4147                 value &= 0xfffeffff;
4148                 writel(value, (void *)real_offset);
4149         }
4150         beiscsi_unmap_pci_function(phba);
4151         pci_free_consistent(phba->pcidev,
4152                             phba->ctrl.mbox_mem_alloced.size,
4153                             phba->ctrl.mbox_mem_alloced.va,
4154                             phba->ctrl.mbox_mem_alloced.dma);
4155         iscsi_boot_destroy_kset(phba->boot_kset);
4156         iscsi_host_remove(phba->shost);
4157         pci_dev_put(phba->pcidev);
4158         iscsi_host_free(phba->shost);
4159 }
4160
4161 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4162 {
4163         int i, status;
4164
4165         for (i = 0; i <= phba->num_cpus; i++)
4166                 phba->msix_entries[i].entry = i;
4167
4168         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4169                                  (phba->num_cpus + 1));
4170         if (!status)
4171                 phba->msix_enabled = true;
4172
4173         return;
4174 }
4175
4176 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4177                                 const struct pci_device_id *id)
4178 {
4179         struct beiscsi_hba *phba = NULL;
4180         struct hwi_controller *phwi_ctrlr;
4181         struct hwi_context_memory *phwi_context;
4182         struct be_eq_obj *pbe_eq;
4183         int ret, num_cpus, i;
4184         u8 *real_offset = 0;
4185         u32 value = 0;
4186
4187         ret = beiscsi_enable_pci(pcidev);
4188         if (ret < 0) {
4189                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4190                         " Failed to enable pci device\n");
4191                 return ret;
4192         }
4193
4194         phba = beiscsi_hba_alloc(pcidev);
4195         if (!phba) {
4196                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4197                         " Failed in beiscsi_hba_alloc\n");
4198                 goto disable_pci;
4199         }
4200
4201         switch (pcidev->device) {
4202         case BE_DEVICE_ID1:
4203         case OC_DEVICE_ID1:
4204         case OC_DEVICE_ID2:
4205                 phba->generation = BE_GEN2;
4206                 break;
4207         case BE_DEVICE_ID2:
4208         case OC_DEVICE_ID3:
4209                 phba->generation = BE_GEN3;
4210                 break;
4211         default:
4212                 phba->generation = 0;
4213         }
4214
4215         if (enable_msix)
4216                 num_cpus = find_num_cpus();
4217         else
4218                 num_cpus = 1;
4219         phba->num_cpus = num_cpus;
4220         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
4221
4222         if (enable_msix)
4223                 beiscsi_msix_enable(phba);
4224         ret = be_ctrl_init(phba, pcidev);
4225         if (ret) {
4226                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4227                                 "Failed in be_ctrl_init\n");
4228                 goto hba_free;
4229         }
4230
4231         if (!num_hba) {
4232                 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4233                 value = readl((void *)real_offset);
4234                 if (value & 0x00010000) {
4235                         gcrashmode++;
4236                         shost_printk(KERN_ERR, phba->shost,
4237                                 "Loading Driver in crashdump mode\n");
4238                         ret = beiscsi_pci_soft_reset(phba);
4239                         if (ret) {
4240                                 shost_printk(KERN_ERR, phba->shost,
4241                                         "Reset Failed. Aborting Crashdump\n");
4242                                 goto hba_free;
4243                         }
4244                         ret = be_chk_reset_complete(phba);
4245                         if (ret) {
4246                                 shost_printk(KERN_ERR, phba->shost,
4247                                         "Failed to get out of reset."
4248                                         "Aborting Crashdump\n");
4249                                 goto hba_free;
4250                         }
4251                 } else {
4252                         value |= 0x00010000;
4253                         writel(value, (void *)real_offset);
4254                         num_hba++;
4255                 }
4256         }
4257
4258         spin_lock_init(&phba->io_sgl_lock);
4259         spin_lock_init(&phba->mgmt_sgl_lock);
4260         spin_lock_init(&phba->isr_lock);
4261         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4262         if (ret != 0) {
4263                 shost_printk(KERN_ERR, phba->shost,
4264                              "Error getting fw config\n");
4265                 goto free_port;
4266         }
4267         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4268         beiscsi_get_params(phba);
4269         phba->shost->can_queue = phba->params.ios_per_ctrl;
4270         ret = beiscsi_init_port(phba);
4271         if (ret < 0) {
4272                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4273                              "Failed in beiscsi_init_port\n");
4274                 goto free_port;
4275         }
4276
4277         for (i = 0; i < MAX_MCC_CMD ; i++) {
4278                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4279                 phba->ctrl.mcc_tag[i] = i + 1;
4280                 phba->ctrl.mcc_numtag[i + 1] = 0;
4281                 phba->ctrl.mcc_tag_available++;
4282         }
4283
4284         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4285
4286         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4287                  phba->shost->host_no);
4288         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4289         if (!phba->wq) {
4290                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4291                                 "Failed to allocate work queue\n");
4292                 goto free_twq;
4293         }
4294
4295         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4296
4297         phwi_ctrlr = phba->phwi_ctrlr;
4298         phwi_context = phwi_ctrlr->phwi_ctxt;
4299         if (blk_iopoll_enabled) {
4300                 for (i = 0; i < phba->num_cpus; i++) {
4301                         pbe_eq = &phwi_context->be_eq[i];
4302                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4303                                         be_iopoll);
4304                         blk_iopoll_enable(&pbe_eq->iopoll);
4305                 }
4306         }
4307         ret = beiscsi_init_irqs(phba);
4308         if (ret < 0) {
4309                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4310                              "Failed to beiscsi_init_irqs\n");
4311                 goto free_blkenbld;
4312         }
4313         hwi_enable_intr(phba);
4314
4315         if (beiscsi_setup_boot_info(phba))
4316                 /*
4317                  * log error but continue, because we may not be using
4318                  * iscsi boot.
4319                  */
4320                 shost_printk(KERN_ERR, phba->shost, "Could not set up "
4321                              "iSCSI boot info.");
4322
4323         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4324         return 0;
4325
4326 free_blkenbld:
4327         destroy_workqueue(phba->wq);
4328         if (blk_iopoll_enabled)
4329                 for (i = 0; i < phba->num_cpus; i++) {
4330                         pbe_eq = &phwi_context->be_eq[i];
4331                         blk_iopoll_disable(&pbe_eq->iopoll);
4332                 }
4333 free_twq:
4334         beiscsi_clean_port(phba);
4335         beiscsi_free_mem(phba);
4336 free_port:
4337         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4338
4339         value = readl((void *)real_offset);
4340
4341         if (value & 0x00010000) {
4342                 value &= 0xfffeffff;
4343                 writel(value, (void *)real_offset);
4344         }
4345
4346         pci_free_consistent(phba->pcidev,
4347                             phba->ctrl.mbox_mem_alloced.size,
4348                             phba->ctrl.mbox_mem_alloced.va,
4349                            phba->ctrl.mbox_mem_alloced.dma);
4350         beiscsi_unmap_pci_function(phba);
4351 hba_free:
4352         if (phba->msix_enabled)
4353                 pci_disable_msix(phba->pcidev);
4354         iscsi_host_remove(phba->shost);
4355         pci_dev_put(phba->pcidev);
4356         iscsi_host_free(phba->shost);
4357 disable_pci:
4358         pci_disable_device(pcidev);
4359         return ret;
4360 }
4361
4362 struct iscsi_transport beiscsi_iscsi_transport = {
4363         .owner = THIS_MODULE,
4364         .name = DRV_NAME,
4365         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4366                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4367         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
4368                                 ISCSI_HOST_INITIATOR_NAME,
4369         .create_session = beiscsi_session_create,
4370         .destroy_session = beiscsi_session_destroy,
4371         .create_conn = beiscsi_conn_create,
4372         .bind_conn = beiscsi_conn_bind,
4373         .destroy_conn = iscsi_conn_teardown,
4374         .attr_is_visible = be2iscsi_attr_is_visible,
4375         .set_param = beiscsi_set_param,
4376         .get_conn_param = iscsi_conn_get_param,
4377         .get_session_param = iscsi_session_get_param,
4378         .get_host_param = beiscsi_get_host_param,
4379         .start_conn = beiscsi_conn_start,
4380         .stop_conn = iscsi_conn_stop,
4381         .send_pdu = iscsi_conn_send_pdu,
4382         .xmit_task = beiscsi_task_xmit,
4383         .cleanup_task = beiscsi_cleanup_task,
4384         .alloc_pdu = beiscsi_alloc_pdu,
4385         .parse_pdu_itt = beiscsi_parse_pdu,
4386         .get_stats = beiscsi_conn_get_stats,
4387         .get_ep_param = beiscsi_ep_get_param,
4388         .ep_connect = beiscsi_ep_connect,
4389         .ep_poll = beiscsi_ep_poll,
4390         .ep_disconnect = beiscsi_ep_disconnect,
4391         .session_recovery_timedout = iscsi_session_recovery_timedout,
4392 };
4393
4394 static struct pci_driver beiscsi_pci_driver = {
4395         .name = DRV_NAME,
4396         .probe = beiscsi_dev_probe,
4397         .remove = beiscsi_remove,
4398         .id_table = beiscsi_pci_id_table
4399 };
4400
4401
4402 static int __init beiscsi_module_init(void)
4403 {
4404         int ret;
4405
4406         beiscsi_scsi_transport =
4407                         iscsi_register_transport(&beiscsi_iscsi_transport);
4408         if (!beiscsi_scsi_transport) {
4409                 SE_DEBUG(DBG_LVL_1,
4410                          "beiscsi_module_init - Unable to  register beiscsi"
4411                          "transport.\n");
4412                 return -ENOMEM;
4413         }
4414         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
4415                  &beiscsi_iscsi_transport);
4416
4417         ret = pci_register_driver(&beiscsi_pci_driver);
4418         if (ret) {
4419                 SE_DEBUG(DBG_LVL_1,
4420                          "beiscsi_module_init - Unable to  register"
4421                          "beiscsi pci driver.\n");
4422                 goto unregister_iscsi_transport;
4423         }
4424         return 0;
4425
4426 unregister_iscsi_transport:
4427         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4428         return ret;
4429 }
4430
4431 static void __exit beiscsi_module_exit(void)
4432 {
4433         pci_unregister_driver(&beiscsi_pci_driver);
4434         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4435 }
4436
4437 module_init(beiscsi_module_init);
4438 module_exit(beiscsi_module_exit);