[SCSI] be2iscsi: No requirement for endianess change for data_count
[linux-2.6.git] / drivers / scsi / be2iscsi / be_main.c
1 /**
2  * Copyright (C) 2005 - 2009 ServerEngines
3  * All rights reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License version 2
7  * as published by the Free Software Foundation.  The full GNU General
8  * Public License is included in this distribution in the file called COPYING.
9  *
10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
11  *
12  * Contact Information:
13  * linux-drivers@serverengines.com
14  *
15  *  ServerEngines
16  * 209 N. Fair Oaks Ave
17  * Sunnyvale, CA 94085
18  *
19  */
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/interrupt.h>
23 #include <linux/blkdev.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/kernel.h>
27 #include <linux/semaphore.h>
28
29 #include <scsi/libiscsi.h>
30 #include <scsi/scsi_transport_iscsi.h>
31 #include <scsi/scsi_transport.h>
32 #include <scsi/scsi_cmnd.h>
33 #include <scsi/scsi_device.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi.h>
36 #include "be_main.h"
37 #include "be_iscsi.h"
38 #include "be_mgmt.h"
39
40 static unsigned int be_iopoll_budget = 10;
41 static unsigned int be_max_phys_size = 64;
42 static unsigned int enable_msix = 1;
43 static unsigned int ring_mode;
44
45 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
46 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
47 MODULE_AUTHOR("ServerEngines Corporation");
48 MODULE_LICENSE("GPL");
49 module_param(be_iopoll_budget, int, 0);
50 module_param(enable_msix, int, 0);
51 module_param(be_max_phys_size, uint, S_IRUGO);
52 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
53                                    "contiguous memory that can be allocated."
54                                    "Range is 16 - 128");
55
56 static int beiscsi_slave_configure(struct scsi_device *sdev)
57 {
58         blk_queue_max_segment_size(sdev->request_queue, 65536);
59         return 0;
60 }
61
62 /*------------------- PCI Driver operations and data ----------------- */
63 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
64         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
65         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
66         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
67         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
68         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID4) },
69         { 0 }
70 };
71 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
72
73 static struct scsi_host_template beiscsi_sht = {
74         .module = THIS_MODULE,
75         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
76         .proc_name = DRV_NAME,
77         .queuecommand = iscsi_queuecommand,
78         .eh_abort_handler = iscsi_eh_abort,
79         .change_queue_depth = iscsi_change_queue_depth,
80         .slave_configure = beiscsi_slave_configure,
81         .target_alloc = iscsi_target_alloc,
82         .eh_device_reset_handler = iscsi_eh_device_reset,
83         .eh_target_reset_handler = iscsi_eh_target_reset,
84         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
85         .can_queue = BE2_IO_DEPTH,
86         .this_id = -1,
87         .max_sectors = BEISCSI_MAX_SECTORS,
88         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
89         .use_clustering = ENABLE_CLUSTERING,
90 };
91
92 static struct scsi_transport_template *beiscsi_scsi_transport;
93
94 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
95 {
96         struct beiscsi_hba *phba;
97         struct Scsi_Host *shost;
98
99         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
100         if (!shost) {
101                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
102                         "iscsi_host_alloc failed \n");
103                 return NULL;
104         }
105         shost->dma_boundary = pcidev->dma_mask;
106         shost->max_id = BE2_MAX_SESSIONS;
107         shost->max_channel = 0;
108         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
109         shost->max_lun = BEISCSI_NUM_MAX_LUN;
110         shost->transportt = beiscsi_scsi_transport;
111         phba = iscsi_host_priv(shost);
112         memset(phba, 0, sizeof(*phba));
113         phba->shost = shost;
114         phba->pcidev = pci_dev_get(pcidev);
115         pci_set_drvdata(pcidev, phba);
116
117         if (iscsi_host_add(shost, &phba->pcidev->dev))
118                 goto free_devices;
119         return phba;
120
121 free_devices:
122         pci_dev_put(phba->pcidev);
123         iscsi_host_free(phba->shost);
124         return NULL;
125 }
126
127 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
128 {
129         if (phba->csr_va) {
130                 iounmap(phba->csr_va);
131                 phba->csr_va = NULL;
132         }
133         if (phba->db_va) {
134                 iounmap(phba->db_va);
135                 phba->db_va = NULL;
136         }
137         if (phba->pci_va) {
138                 iounmap(phba->pci_va);
139                 phba->pci_va = NULL;
140         }
141 }
142
143 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
144                                 struct pci_dev *pcidev)
145 {
146         u8 __iomem *addr;
147
148         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
149                                pci_resource_len(pcidev, 2));
150         if (addr == NULL)
151                 return -ENOMEM;
152         phba->ctrl.csr = addr;
153         phba->csr_va = addr;
154         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
155
156         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
157         if (addr == NULL)
158                 goto pci_map_err;
159         phba->ctrl.db = addr;
160         phba->db_va = addr;
161         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
162
163         addr = ioremap_nocache(pci_resource_start(pcidev, 1),
164                                pci_resource_len(pcidev, 1));
165         if (addr == NULL)
166                 goto pci_map_err;
167         phba->ctrl.pcicfg = addr;
168         phba->pci_va = addr;
169         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
170         return 0;
171
172 pci_map_err:
173         beiscsi_unmap_pci_function(phba);
174         return -ENOMEM;
175 }
176
177 static int beiscsi_enable_pci(struct pci_dev *pcidev)
178 {
179         int ret;
180
181         ret = pci_enable_device(pcidev);
182         if (ret) {
183                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
184                         "failed. Returning -ENODEV\n");
185                 return ret;
186         }
187
188         pci_set_master(pcidev);
189         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
190                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
191                 if (ret) {
192                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
193                         pci_disable_device(pcidev);
194                         return ret;
195                 }
196         }
197         return 0;
198 }
199
200 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
201 {
202         struct be_ctrl_info *ctrl = &phba->ctrl;
203         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
204         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
205         int status = 0;
206
207         ctrl->pdev = pdev;
208         status = beiscsi_map_pci_bars(phba, pdev);
209         if (status)
210                 return status;
211         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
212         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
213                                                   mbox_mem_alloc->size,
214                                                   &mbox_mem_alloc->dma);
215         if (!mbox_mem_alloc->va) {
216                 beiscsi_unmap_pci_function(phba);
217                 status = -ENOMEM;
218                 return status;
219         }
220
221         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
222         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
223         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
224         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
225         spin_lock_init(&ctrl->mbox_lock);
226         spin_lock_init(&phba->ctrl.mcc_lock);
227         spin_lock_init(&phba->ctrl.mcc_cq_lock);
228
229         return status;
230 }
231
232 static void beiscsi_get_params(struct beiscsi_hba *phba)
233 {
234         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
235                                     - (phba->fw_config.iscsi_cid_count
236                                     + BE2_TMFS
237                                     + BE2_NOPOUT_REQ));
238         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
239         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count;;
240         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;;
241         phba->params.num_sge_per_io = BE2_SGE;
242         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
243         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
244         phba->params.eq_timer = 64;
245         phba->params.num_eq_entries =
246             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
247                                     + BE2_TMFS) / 512) + 1) * 512;
248         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
249                                 ? 1024 : phba->params.num_eq_entries;
250         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
251                              phba->params.num_eq_entries);
252         phba->params.num_cq_entries =
253             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
254                                     + BE2_TMFS) / 512) + 1) * 512;
255         phba->params.wrbs_per_cxn = 256;
256 }
257
258 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
259                            unsigned int id, unsigned int clr_interrupt,
260                            unsigned int num_processed,
261                            unsigned char rearm, unsigned char event)
262 {
263         u32 val = 0;
264         val |= id & DB_EQ_RING_ID_MASK;
265         if (rearm)
266                 val |= 1 << DB_EQ_REARM_SHIFT;
267         if (clr_interrupt)
268                 val |= 1 << DB_EQ_CLR_SHIFT;
269         if (event)
270                 val |= 1 << DB_EQ_EVNT_SHIFT;
271         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
272         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
273 }
274
275 /**
276  * be_isr_mcc - The isr routine of the driver.
277  * @irq: Not used
278  * @dev_id: Pointer to host adapter structure
279  */
280 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
281 {
282         struct beiscsi_hba *phba;
283         struct be_eq_entry *eqe = NULL;
284         struct be_queue_info *eq;
285         struct be_queue_info *mcc;
286         unsigned int num_eq_processed;
287         struct be_eq_obj *pbe_eq;
288         unsigned long flags;
289
290         pbe_eq = dev_id;
291         eq = &pbe_eq->q;
292         phba =  pbe_eq->phba;
293         mcc = &phba->ctrl.mcc_obj.cq;
294         eqe = queue_tail_node(eq);
295         if (!eqe)
296                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
297
298         num_eq_processed = 0;
299
300         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
301                                 & EQE_VALID_MASK) {
302                 if (((eqe->dw[offsetof(struct amap_eq_entry,
303                      resource_id) / 32] &
304                      EQE_RESID_MASK) >> 16) == mcc->id) {
305                         spin_lock_irqsave(&phba->isr_lock, flags);
306                         phba->todo_mcc_cq = 1;
307                         spin_unlock_irqrestore(&phba->isr_lock, flags);
308                 }
309                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
310                 queue_tail_inc(eq);
311                 eqe = queue_tail_node(eq);
312                 num_eq_processed++;
313         }
314         if (phba->todo_mcc_cq)
315                 queue_work(phba->wq, &phba->work_cqs);
316         if (num_eq_processed)
317                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
318
319         return IRQ_HANDLED;
320 }
321
322 /**
323  * be_isr_msix - The isr routine of the driver.
324  * @irq: Not used
325  * @dev_id: Pointer to host adapter structure
326  */
327 static irqreturn_t be_isr_msix(int irq, void *dev_id)
328 {
329         struct beiscsi_hba *phba;
330         struct be_eq_entry *eqe = NULL;
331         struct be_queue_info *eq;
332         struct be_queue_info *cq;
333         unsigned int num_eq_processed;
334         struct be_eq_obj *pbe_eq;
335         unsigned long flags;
336
337         pbe_eq = dev_id;
338         eq = &pbe_eq->q;
339         cq = pbe_eq->cq;
340         eqe = queue_tail_node(eq);
341         if (!eqe)
342                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
343
344         phba = pbe_eq->phba;
345         num_eq_processed = 0;
346         if (blk_iopoll_enabled) {
347                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
348                                         & EQE_VALID_MASK) {
349                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
350                                 blk_iopoll_sched(&pbe_eq->iopoll);
351
352                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
353                         queue_tail_inc(eq);
354                         eqe = queue_tail_node(eq);
355                         num_eq_processed++;
356                 }
357                 if (num_eq_processed)
358                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
359
360                 return IRQ_HANDLED;
361         } else {
362                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
363                                                 & EQE_VALID_MASK) {
364                         spin_lock_irqsave(&phba->isr_lock, flags);
365                         phba->todo_cq = 1;
366                         spin_unlock_irqrestore(&phba->isr_lock, flags);
367                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
368                         queue_tail_inc(eq);
369                         eqe = queue_tail_node(eq);
370                         num_eq_processed++;
371                 }
372                 if (phba->todo_cq)
373                         queue_work(phba->wq, &phba->work_cqs);
374
375                 if (num_eq_processed)
376                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
377
378                 return IRQ_HANDLED;
379         }
380 }
381
382 /**
383  * be_isr - The isr routine of the driver.
384  * @irq: Not used
385  * @dev_id: Pointer to host adapter structure
386  */
387 static irqreturn_t be_isr(int irq, void *dev_id)
388 {
389         struct beiscsi_hba *phba;
390         struct hwi_controller *phwi_ctrlr;
391         struct hwi_context_memory *phwi_context;
392         struct be_eq_entry *eqe = NULL;
393         struct be_queue_info *eq;
394         struct be_queue_info *cq;
395         struct be_queue_info *mcc;
396         unsigned long flags, index;
397         unsigned int num_mcceq_processed, num_ioeq_processed;
398         struct be_ctrl_info *ctrl;
399         struct be_eq_obj *pbe_eq;
400         int isr;
401
402         phba = dev_id;
403         ctrl = &phba->ctrl;;
404         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
405                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
406         if (!isr)
407                 return IRQ_NONE;
408
409         phwi_ctrlr = phba->phwi_ctrlr;
410         phwi_context = phwi_ctrlr->phwi_ctxt;
411         pbe_eq = &phwi_context->be_eq[0];
412
413         eq = &phwi_context->be_eq[0].q;
414         mcc = &phba->ctrl.mcc_obj.cq;
415         index = 0;
416         eqe = queue_tail_node(eq);
417         if (!eqe)
418                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
419
420         num_ioeq_processed = 0;
421         num_mcceq_processed = 0;
422         if (blk_iopoll_enabled) {
423                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
424                                         & EQE_VALID_MASK) {
425                         if (((eqe->dw[offsetof(struct amap_eq_entry,
426                              resource_id) / 32] &
427                              EQE_RESID_MASK) >> 16) == mcc->id) {
428                                 spin_lock_irqsave(&phba->isr_lock, flags);
429                                 phba->todo_mcc_cq = 1;
430                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
431                                 num_mcceq_processed++;
432                         } else {
433                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
434                                         blk_iopoll_sched(&pbe_eq->iopoll);
435                                 num_ioeq_processed++;
436                         }
437                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
438                         queue_tail_inc(eq);
439                         eqe = queue_tail_node(eq);
440                 }
441                 if (num_ioeq_processed || num_mcceq_processed) {
442                         if (phba->todo_mcc_cq)
443                                 queue_work(phba->wq, &phba->work_cqs);
444
445                 if ((num_mcceq_processed) && (!num_ioeq_processed))
446                                 hwi_ring_eq_db(phba, eq->id, 0,
447                                               (num_ioeq_processed +
448                                                num_mcceq_processed) , 1, 1);
449                         else
450                                 hwi_ring_eq_db(phba, eq->id, 0,
451                                                (num_ioeq_processed +
452                                                 num_mcceq_processed), 0, 1);
453
454                         return IRQ_HANDLED;
455                 } else
456                         return IRQ_NONE;
457         } else {
458                 cq = &phwi_context->be_cq[0];
459                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
460                                                 & EQE_VALID_MASK) {
461
462                         if (((eqe->dw[offsetof(struct amap_eq_entry,
463                              resource_id) / 32] &
464                              EQE_RESID_MASK) >> 16) != cq->id) {
465                                 spin_lock_irqsave(&phba->isr_lock, flags);
466                                 phba->todo_mcc_cq = 1;
467                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
468                         } else {
469                                 spin_lock_irqsave(&phba->isr_lock, flags);
470                                 phba->todo_cq = 1;
471                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
472                         }
473                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
474                         queue_tail_inc(eq);
475                         eqe = queue_tail_node(eq);
476                         num_ioeq_processed++;
477                 }
478                 if (phba->todo_cq || phba->todo_mcc_cq)
479                         queue_work(phba->wq, &phba->work_cqs);
480
481                 if (num_ioeq_processed) {
482                         hwi_ring_eq_db(phba, eq->id, 0,
483                                        num_ioeq_processed, 1, 1);
484                         return IRQ_HANDLED;
485                 } else
486                         return IRQ_NONE;
487         }
488 }
489
490 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
491 {
492         struct pci_dev *pcidev = phba->pcidev;
493         struct hwi_controller *phwi_ctrlr;
494         struct hwi_context_memory *phwi_context;
495         int ret, msix_vec, i = 0;
496         char desc[32];
497
498         phwi_ctrlr = phba->phwi_ctrlr;
499         phwi_context = phwi_ctrlr->phwi_ctxt;
500
501         if (phba->msix_enabled) {
502                 for (i = 0; i < phba->num_cpus; i++) {
503                         sprintf(desc, "beiscsi_msix_%04x", i);
504                         msix_vec = phba->msix_entries[i].vector;
505                         ret = request_irq(msix_vec, be_isr_msix, 0, desc,
506                                           &phwi_context->be_eq[i]);
507                 }
508                 msix_vec = phba->msix_entries[i].vector;
509                 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
510                                   &phwi_context->be_eq[i]);
511         } else {
512                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
513                                   "beiscsi", phba);
514                 if (ret) {
515                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
516                                      "Failed to register irq\\n");
517                         return ret;
518                 }
519         }
520         return 0;
521 }
522
523 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
524                            unsigned int id, unsigned int num_processed,
525                            unsigned char rearm, unsigned char event)
526 {
527         u32 val = 0;
528         val |= id & DB_CQ_RING_ID_MASK;
529         if (rearm)
530                 val |= 1 << DB_CQ_REARM_SHIFT;
531         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
532         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
533 }
534
535 static unsigned int
536 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
537                           struct beiscsi_hba *phba,
538                           unsigned short cid,
539                           struct pdu_base *ppdu,
540                           unsigned long pdu_len,
541                           void *pbuffer, unsigned long buf_len)
542 {
543         struct iscsi_conn *conn = beiscsi_conn->conn;
544         struct iscsi_session *session = conn->session;
545         struct iscsi_task *task;
546         struct beiscsi_io_task *io_task;
547         struct iscsi_hdr *login_hdr;
548
549         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
550                                                 PDUBASE_OPCODE_MASK) {
551         case ISCSI_OP_NOOP_IN:
552                 pbuffer = NULL;
553                 buf_len = 0;
554                 break;
555         case ISCSI_OP_ASYNC_EVENT:
556                 break;
557         case ISCSI_OP_REJECT:
558                 WARN_ON(!pbuffer);
559                 WARN_ON(!(buf_len == 48));
560                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
561                 break;
562         case ISCSI_OP_LOGIN_RSP:
563         case ISCSI_OP_TEXT_RSP:
564                 task = conn->login_task;
565                 io_task = task->dd_data;
566                 login_hdr = (struct iscsi_hdr *)ppdu;
567                 login_hdr->itt = io_task->libiscsi_itt;
568                 break;
569         default:
570                 shost_printk(KERN_WARNING, phba->shost,
571                              "Unrecognized opcode 0x%x in async msg \n",
572                              (ppdu->
573                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
574                                                 & PDUBASE_OPCODE_MASK));
575                 return 1;
576         }
577
578         spin_lock_bh(&session->lock);
579         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
580         spin_unlock_bh(&session->lock);
581         return 0;
582 }
583
584 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
585 {
586         struct sgl_handle *psgl_handle;
587
588         if (phba->io_sgl_hndl_avbl) {
589                 SE_DEBUG(DBG_LVL_8,
590                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
591                          phba->io_sgl_alloc_index);
592                 psgl_handle = phba->io_sgl_hndl_base[phba->
593                                                 io_sgl_alloc_index];
594                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
595                 phba->io_sgl_hndl_avbl--;
596                 if (phba->io_sgl_alloc_index == (phba->params.
597                                                  ios_per_ctrl - 1))
598                         phba->io_sgl_alloc_index = 0;
599                 else
600                         phba->io_sgl_alloc_index++;
601         } else
602                 psgl_handle = NULL;
603         return psgl_handle;
604 }
605
606 static void
607 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
608 {
609         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
610                  phba->io_sgl_free_index);
611         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
612                 /*
613                  * this can happen if clean_task is called on a task that
614                  * failed in xmit_task or alloc_pdu.
615                  */
616                  SE_DEBUG(DBG_LVL_8,
617                          "Double Free in IO SGL io_sgl_free_index=%d,"
618                          "value there=%p \n", phba->io_sgl_free_index,
619                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
620                 return;
621         }
622         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
623         phba->io_sgl_hndl_avbl++;
624         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
625                 phba->io_sgl_free_index = 0;
626         else
627                 phba->io_sgl_free_index++;
628 }
629
630 /**
631  * alloc_wrb_handle - To allocate a wrb handle
632  * @phba: The hba pointer
633  * @cid: The cid to use for allocation
634  *
635  * This happens under session_lock until submission to chip
636  */
637 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
638 {
639         struct hwi_wrb_context *pwrb_context;
640         struct hwi_controller *phwi_ctrlr;
641         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
642
643         phwi_ctrlr = phba->phwi_ctrlr;
644         pwrb_context = &phwi_ctrlr->wrb_context[cid];
645         if (pwrb_context->wrb_handles_available >= 2) {
646                 pwrb_handle = pwrb_context->pwrb_handle_base[
647                                             pwrb_context->alloc_index];
648                 pwrb_context->wrb_handles_available--;
649                 if (pwrb_context->alloc_index ==
650                                                 (phba->params.wrbs_per_cxn - 1))
651                         pwrb_context->alloc_index = 0;
652                 else
653                         pwrb_context->alloc_index++;
654
655                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
656                                                 pwrb_context->alloc_index];
657                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
658         } else
659                 pwrb_handle = NULL;
660         return pwrb_handle;
661 }
662
663 /**
664  * free_wrb_handle - To free the wrb handle back to pool
665  * @phba: The hba pointer
666  * @pwrb_context: The context to free from
667  * @pwrb_handle: The wrb_handle to free
668  *
669  * This happens under session_lock until submission to chip
670  */
671 static void
672 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
673                 struct wrb_handle *pwrb_handle)
674 {
675         if (!ring_mode)
676                 pwrb_context->pwrb_handle_base[pwrb_context->free_index] =
677                                                pwrb_handle;
678         pwrb_context->wrb_handles_available++;
679         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
680                 pwrb_context->free_index = 0;
681         else
682                 pwrb_context->free_index++;
683
684         SE_DEBUG(DBG_LVL_8,
685                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
686                  "wrb_handles_available=%d \n",
687                  pwrb_handle, pwrb_context->free_index,
688                  pwrb_context->wrb_handles_available);
689 }
690
691 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
692 {
693         struct sgl_handle *psgl_handle;
694
695         if (phba->eh_sgl_hndl_avbl) {
696                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
697                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
698                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
699                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
700                 phba->eh_sgl_hndl_avbl--;
701                 if (phba->eh_sgl_alloc_index ==
702                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
703                      1))
704                         phba->eh_sgl_alloc_index = 0;
705                 else
706                         phba->eh_sgl_alloc_index++;
707         } else
708                 psgl_handle = NULL;
709         return psgl_handle;
710 }
711
712 void
713 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
714 {
715
716         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
717                              phba->eh_sgl_free_index);
718         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
719                 /*
720                  * this can happen if clean_task is called on a task that
721                  * failed in xmit_task or alloc_pdu.
722                  */
723                 SE_DEBUG(DBG_LVL_8,
724                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
725                          phba->eh_sgl_free_index);
726                 return;
727         }
728         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
729         phba->eh_sgl_hndl_avbl++;
730         if (phba->eh_sgl_free_index ==
731             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
732                 phba->eh_sgl_free_index = 0;
733         else
734                 phba->eh_sgl_free_index++;
735 }
736
737 static void
738 be_complete_io(struct beiscsi_conn *beiscsi_conn,
739                struct iscsi_task *task, struct sol_cqe *psol)
740 {
741         struct beiscsi_io_task *io_task = task->dd_data;
742         struct be_status_bhs *sts_bhs =
743                                 (struct be_status_bhs *)io_task->cmd_bhs;
744         struct iscsi_conn *conn = beiscsi_conn->conn;
745         unsigned int sense_len;
746         unsigned char *sense;
747         u32 resid = 0, exp_cmdsn, max_cmdsn;
748         u8 rsp, status, flags;
749
750         exp_cmdsn = (psol->
751                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
752                         & SOL_EXP_CMD_SN_MASK);
753         max_cmdsn = ((psol->
754                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
755                         & SOL_EXP_CMD_SN_MASK) +
756                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
757                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
758         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
759                                                 & SOL_RESP_MASK) >> 16);
760         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
761                                                 & SOL_STS_MASK) >> 8);
762         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
763                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
764
765         task->sc->result = (DID_OK << 16) | status;
766         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
767                 task->sc->result = DID_ERROR << 16;
768                 goto unmap;
769         }
770
771         /* bidi not initially supported */
772         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
773                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
774                                 32] & SOL_RES_CNT_MASK);
775
776                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
777                         task->sc->result = DID_ERROR << 16;
778
779                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
780                         scsi_set_resid(task->sc, resid);
781                         if (!status && (scsi_bufflen(task->sc) - resid <
782                             task->sc->underflow))
783                                 task->sc->result = DID_ERROR << 16;
784                 }
785         }
786
787         if (status == SAM_STAT_CHECK_CONDITION) {
788                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
789                 sense = sts_bhs->sense_info + sizeof(unsigned short);
790                 sense_len =  cpu_to_be16(*slen);
791                 memcpy(task->sc->sense_buffer, sense,
792                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
793         }
794         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
795                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
796                                                         & SOL_RES_CNT_MASK)
797                          conn->rxdata_octets += (psol->
798                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
799                              & SOL_RES_CNT_MASK);
800         }
801 unmap:
802         scsi_dma_unmap(io_task->scsi_cmnd);
803         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
804 }
805
806 static void
807 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
808                    struct iscsi_task *task, struct sol_cqe *psol)
809 {
810         struct iscsi_logout_rsp *hdr;
811         struct beiscsi_io_task *io_task = task->dd_data;
812         struct iscsi_conn *conn = beiscsi_conn->conn;
813
814         hdr = (struct iscsi_logout_rsp *)task->hdr;
815         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
816         hdr->t2wait = 5;
817         hdr->t2retain = 0;
818         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
819                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
820         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
821                                         32] & SOL_RESP_MASK);
822         hdr->exp_cmdsn = cpu_to_be32(psol->
823                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
824                                         & SOL_EXP_CMD_SN_MASK);
825         hdr->max_cmdsn = be32_to_cpu((psol->
826                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
827                                         & SOL_EXP_CMD_SN_MASK) +
828                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
829                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
830         hdr->dlength[0] = 0;
831         hdr->dlength[1] = 0;
832         hdr->dlength[2] = 0;
833         hdr->hlength = 0;
834         hdr->itt = io_task->libiscsi_itt;
835         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
836 }
837
838 static void
839 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
840                 struct iscsi_task *task, struct sol_cqe *psol)
841 {
842         struct iscsi_tm_rsp *hdr;
843         struct iscsi_conn *conn = beiscsi_conn->conn;
844         struct beiscsi_io_task *io_task = task->dd_data;
845
846         hdr = (struct iscsi_tm_rsp *)task->hdr;
847         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
848         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
849                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
850         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
851                                         32] & SOL_RESP_MASK);
852         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
853                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
854         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
855                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
856                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
857                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
858         hdr->itt = io_task->libiscsi_itt;
859         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
860 }
861
862 static void
863 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
864                        struct beiscsi_hba *phba, struct sol_cqe *psol)
865 {
866         struct hwi_wrb_context *pwrb_context;
867         struct wrb_handle *pwrb_handle = NULL;
868         struct sgl_handle *psgl_handle = NULL;
869         struct hwi_controller *phwi_ctrlr;
870         struct iscsi_task *task;
871         struct beiscsi_io_task *io_task;
872         struct iscsi_conn *conn = beiscsi_conn->conn;
873         struct iscsi_session *session = conn->session;
874
875         phwi_ctrlr = phba->phwi_ctrlr;
876         if (ring_mode) {
877                 psgl_handle = phba->sgl_hndl_array[((psol->
878                               dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
879                                 32] & SOL_ICD_INDEX_MASK) >> 6)];
880                 pwrb_context = &phwi_ctrlr->wrb_context[psgl_handle->cid];
881                 task = psgl_handle->task;
882                 pwrb_handle = NULL;
883         } else {
884                 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
885                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
886                                 SOL_CID_MASK) >> 6) -
887                                 phba->fw_config.iscsi_cid_start];
888                 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
889                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
890                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
891                 task = pwrb_handle->pio_handle;
892         }
893
894         io_task = task->dd_data;
895         spin_lock(&phba->mgmt_sgl_lock);
896         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
897         spin_unlock(&phba->mgmt_sgl_lock);
898         spin_lock_bh(&session->lock);
899         free_wrb_handle(phba, pwrb_context, pwrb_handle);
900         spin_unlock_bh(&session->lock);
901 }
902
903 static void
904 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
905                        struct iscsi_task *task, struct sol_cqe *psol)
906 {
907         struct iscsi_nopin *hdr;
908         struct iscsi_conn *conn = beiscsi_conn->conn;
909         struct beiscsi_io_task *io_task = task->dd_data;
910
911         hdr = (struct iscsi_nopin *)task->hdr;
912         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
913                         & SOL_FLAGS_MASK) >> 24) | 0x80;
914         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
915                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
916         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
917                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
918                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
919                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
920         hdr->opcode = ISCSI_OP_NOOP_IN;
921         hdr->itt = io_task->libiscsi_itt;
922         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
923 }
924
925 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
926                              struct beiscsi_hba *phba, struct sol_cqe *psol)
927 {
928         struct hwi_wrb_context *pwrb_context;
929         struct wrb_handle *pwrb_handle;
930         struct iscsi_wrb *pwrb = NULL;
931         struct hwi_controller *phwi_ctrlr;
932         struct iscsi_task *task;
933         struct sgl_handle *psgl_handle = NULL;
934         unsigned int type;
935         struct iscsi_conn *conn = beiscsi_conn->conn;
936         struct iscsi_session *session = conn->session;
937
938         phwi_ctrlr = phba->phwi_ctrlr;
939         if (ring_mode) {
940                 psgl_handle = phba->sgl_hndl_array[((psol->
941                               dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
942                               32] & SOL_ICD_INDEX_MASK) >> 6)];
943                 task = psgl_handle->task;
944                 type = psgl_handle->type;
945         } else {
946                 pwrb_context = &phwi_ctrlr->
947                                 wrb_context[((psol->dw[offsetof
948                                 (struct amap_sol_cqe, cid) / 32]
949                                 & SOL_CID_MASK) >> 6) -
950                                 phba->fw_config.iscsi_cid_start];
951                 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
952                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
953                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
954                 task = pwrb_handle->pio_handle;
955                 pwrb = pwrb_handle->pwrb;
956                 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
957                          WRB_TYPE_MASK) >> 28;
958         }
959         spin_lock_bh(&session->lock);
960         switch (type) {
961         case HWH_TYPE_IO:
962         case HWH_TYPE_IO_RD:
963                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
964                     ISCSI_OP_NOOP_OUT) {
965                         be_complete_nopin_resp(beiscsi_conn, task, psol);
966                 } else
967                         be_complete_io(beiscsi_conn, task, psol);
968                 break;
969
970         case HWH_TYPE_LOGOUT:
971                 be_complete_logout(beiscsi_conn, task, psol);
972                 break;
973
974         case HWH_TYPE_LOGIN:
975                 SE_DEBUG(DBG_LVL_1,
976                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
977                          "- Solicited path \n");
978                 break;
979
980         case HWH_TYPE_TMF:
981                 be_complete_tmf(beiscsi_conn, task, psol);
982                 break;
983
984         case HWH_TYPE_NOP:
985                 be_complete_nopin_resp(beiscsi_conn, task, psol);
986                 break;
987
988         default:
989                 if (ring_mode)
990                         shost_printk(KERN_WARNING, phba->shost,
991                                 "In hwi_complete_cmd, unknown type = %d"
992                                 "icd_index 0x%x CID 0x%x\n", type,
993                                 ((psol->dw[offsetof(struct amap_sol_cqe_ring,
994                                 icd_index) / 32] & SOL_ICD_INDEX_MASK) >> 6),
995                                 psgl_handle->cid);
996                 else
997                         shost_printk(KERN_WARNING, phba->shost,
998                                 "In hwi_complete_cmd, unknown type = %d"
999                                 "wrb_index 0x%x CID 0x%x\n", type,
1000                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1001                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1002                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1003                                 cid) / 32] & SOL_CID_MASK) >> 6));
1004                 break;
1005         }
1006
1007         spin_unlock_bh(&session->lock);
1008 }
1009
1010 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1011                                           *pasync_ctx, unsigned int is_header,
1012                                           unsigned int host_write_ptr)
1013 {
1014         if (is_header)
1015                 return &pasync_ctx->async_entry[host_write_ptr].
1016                     header_busy_list;
1017         else
1018                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1019 }
1020
1021 static struct async_pdu_handle *
1022 hwi_get_async_handle(struct beiscsi_hba *phba,
1023                      struct beiscsi_conn *beiscsi_conn,
1024                      struct hwi_async_pdu_context *pasync_ctx,
1025                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1026 {
1027         struct be_bus_address phys_addr;
1028         struct list_head *pbusy_list;
1029         struct async_pdu_handle *pasync_handle = NULL;
1030         int buffer_len = 0;
1031         unsigned char buffer_index = -1;
1032         unsigned char is_header = 0;
1033
1034         phys_addr.u.a32.address_lo =
1035             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1036             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1037                                                 & PDUCQE_DPL_MASK) >> 16);
1038         phys_addr.u.a32.address_hi =
1039             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1040
1041         phys_addr.u.a64.address =
1042                         *((unsigned long long *)(&phys_addr.u.a64.address));
1043
1044         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1045                         & PDUCQE_CODE_MASK) {
1046         case UNSOL_HDR_NOTIFY:
1047                 is_header = 1;
1048
1049                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1050                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1051                         index) / 32] & PDUCQE_INDEX_MASK));
1052
1053                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1054                                 pasync_ctx->async_header.pa_base.u.a64.address);
1055
1056                 buffer_index = buffer_len /
1057                                 pasync_ctx->async_header.buffer_size;
1058
1059                 break;
1060         case UNSOL_DATA_NOTIFY:
1061                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1062                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1063                                         index) / 32] & PDUCQE_INDEX_MASK));
1064                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1065                                         pasync_ctx->async_data.pa_base.u.
1066                                         a64.address);
1067                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1068                 break;
1069         default:
1070                 pbusy_list = NULL;
1071                 shost_printk(KERN_WARNING, phba->shost,
1072                         "Unexpected code=%d \n",
1073                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1074                                         code) / 32] & PDUCQE_CODE_MASK);
1075                 return NULL;
1076         }
1077
1078         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1079         WARN_ON(list_empty(pbusy_list));
1080         list_for_each_entry(pasync_handle, pbusy_list, link) {
1081                 WARN_ON(pasync_handle->consumed);
1082                 if (pasync_handle->index == buffer_index)
1083                         break;
1084         }
1085
1086         WARN_ON(!pasync_handle);
1087
1088         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1089                                              phba->fw_config.iscsi_cid_start;
1090         pasync_handle->is_header = is_header;
1091         pasync_handle->buffer_len = ((pdpdu_cqe->
1092                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1093                         & PDUCQE_DPL_MASK) >> 16);
1094
1095         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1096                         index) / 32] & PDUCQE_INDEX_MASK);
1097         return pasync_handle;
1098 }
1099
1100 static unsigned int
1101 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1102                            unsigned int is_header, unsigned int cq_index)
1103 {
1104         struct list_head *pbusy_list;
1105         struct async_pdu_handle *pasync_handle;
1106         unsigned int num_entries, writables = 0;
1107         unsigned int *pep_read_ptr, *pwritables;
1108
1109
1110         if (is_header) {
1111                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1112                 pwritables = &pasync_ctx->async_header.writables;
1113                 num_entries = pasync_ctx->async_header.num_entries;
1114         } else {
1115                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1116                 pwritables = &pasync_ctx->async_data.writables;
1117                 num_entries = pasync_ctx->async_data.num_entries;
1118         }
1119
1120         while ((*pep_read_ptr) != cq_index) {
1121                 (*pep_read_ptr)++;
1122                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1123
1124                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1125                                                      *pep_read_ptr);
1126                 if (writables == 0)
1127                         WARN_ON(list_empty(pbusy_list));
1128
1129                 if (!list_empty(pbusy_list)) {
1130                         pasync_handle = list_entry(pbusy_list->next,
1131                                                    struct async_pdu_handle,
1132                                                    link);
1133                         WARN_ON(!pasync_handle);
1134                         pasync_handle->consumed = 1;
1135                 }
1136
1137                 writables++;
1138         }
1139
1140         if (!writables) {
1141                 SE_DEBUG(DBG_LVL_1,
1142                          "Duplicate notification received - index 0x%x!!\n",
1143                          cq_index);
1144                 WARN_ON(1);
1145         }
1146
1147         *pwritables = *pwritables + writables;
1148         return 0;
1149 }
1150
1151 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1152                                        unsigned int cri)
1153 {
1154         struct hwi_controller *phwi_ctrlr;
1155         struct hwi_async_pdu_context *pasync_ctx;
1156         struct async_pdu_handle *pasync_handle, *tmp_handle;
1157         struct list_head *plist;
1158         unsigned int i = 0;
1159
1160         phwi_ctrlr = phba->phwi_ctrlr;
1161         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1162
1163         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1164
1165         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1166                 list_del(&pasync_handle->link);
1167
1168                 if (i == 0) {
1169                         list_add_tail(&pasync_handle->link,
1170                                       &pasync_ctx->async_header.free_list);
1171                         pasync_ctx->async_header.free_entries++;
1172                         i++;
1173                 } else {
1174                         list_add_tail(&pasync_handle->link,
1175                                       &pasync_ctx->async_data.free_list);
1176                         pasync_ctx->async_data.free_entries++;
1177                         i++;
1178                 }
1179         }
1180
1181         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1182         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1183         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1184         return 0;
1185 }
1186
1187 static struct phys_addr *
1188 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1189                      unsigned int is_header, unsigned int host_write_ptr)
1190 {
1191         struct phys_addr *pasync_sge = NULL;
1192
1193         if (is_header)
1194                 pasync_sge = pasync_ctx->async_header.ring_base;
1195         else
1196                 pasync_sge = pasync_ctx->async_data.ring_base;
1197
1198         return pasync_sge + host_write_ptr;
1199 }
1200
1201 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1202                                    unsigned int is_header)
1203 {
1204         struct hwi_controller *phwi_ctrlr;
1205         struct hwi_async_pdu_context *pasync_ctx;
1206         struct async_pdu_handle *pasync_handle;
1207         struct list_head *pfree_link, *pbusy_list;
1208         struct phys_addr *pasync_sge;
1209         unsigned int ring_id, num_entries;
1210         unsigned int host_write_num;
1211         unsigned int writables;
1212         unsigned int i = 0;
1213         u32 doorbell = 0;
1214
1215         phwi_ctrlr = phba->phwi_ctrlr;
1216         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1217
1218         if (is_header) {
1219                 num_entries = pasync_ctx->async_header.num_entries;
1220                 writables = min(pasync_ctx->async_header.writables,
1221                                 pasync_ctx->async_header.free_entries);
1222                 pfree_link = pasync_ctx->async_header.free_list.next;
1223                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1224                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1225         } else {
1226                 num_entries = pasync_ctx->async_data.num_entries;
1227                 writables = min(pasync_ctx->async_data.writables,
1228                                 pasync_ctx->async_data.free_entries);
1229                 pfree_link = pasync_ctx->async_data.free_list.next;
1230                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1231                 ring_id = phwi_ctrlr->default_pdu_data.id;
1232         }
1233
1234         writables = (writables / 8) * 8;
1235         if (writables) {
1236                 for (i = 0; i < writables; i++) {
1237                         pbusy_list =
1238                             hwi_get_async_busy_list(pasync_ctx, is_header,
1239                                                     host_write_num);
1240                         pasync_handle =
1241                             list_entry(pfree_link, struct async_pdu_handle,
1242                                                                 link);
1243                         WARN_ON(!pasync_handle);
1244                         pasync_handle->consumed = 0;
1245
1246                         pfree_link = pfree_link->next;
1247
1248                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1249                                                 is_header, host_write_num);
1250
1251                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1252                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1253
1254                         list_move(&pasync_handle->link, pbusy_list);
1255
1256                         host_write_num++;
1257                         host_write_num = host_write_num % num_entries;
1258                 }
1259
1260                 if (is_header) {
1261                         pasync_ctx->async_header.host_write_ptr =
1262                                                         host_write_num;
1263                         pasync_ctx->async_header.free_entries -= writables;
1264                         pasync_ctx->async_header.writables -= writables;
1265                         pasync_ctx->async_header.busy_entries += writables;
1266                 } else {
1267                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1268                         pasync_ctx->async_data.free_entries -= writables;
1269                         pasync_ctx->async_data.writables -= writables;
1270                         pasync_ctx->async_data.busy_entries += writables;
1271                 }
1272
1273                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1274                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1275                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1276                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1277                                         << DB_DEF_PDU_CQPROC_SHIFT;
1278
1279                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1280         }
1281 }
1282
1283 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1284                                          struct beiscsi_conn *beiscsi_conn,
1285                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1286 {
1287         struct hwi_controller *phwi_ctrlr;
1288         struct hwi_async_pdu_context *pasync_ctx;
1289         struct async_pdu_handle *pasync_handle = NULL;
1290         unsigned int cq_index = -1;
1291
1292         phwi_ctrlr = phba->phwi_ctrlr;
1293         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1294
1295         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1296                                              pdpdu_cqe, &cq_index);
1297         BUG_ON(pasync_handle->is_header != 0);
1298         if (pasync_handle->consumed == 0)
1299                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1300                                            cq_index);
1301
1302         hwi_free_async_msg(phba, pasync_handle->cri);
1303         hwi_post_async_buffers(phba, pasync_handle->is_header);
1304 }
1305
1306 static unsigned int
1307 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1308                   struct beiscsi_hba *phba,
1309                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1310 {
1311         struct list_head *plist;
1312         struct async_pdu_handle *pasync_handle;
1313         void *phdr = NULL;
1314         unsigned int hdr_len = 0, buf_len = 0;
1315         unsigned int status, index = 0, offset = 0;
1316         void *pfirst_buffer = NULL;
1317         unsigned int num_buf = 0;
1318
1319         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1320
1321         list_for_each_entry(pasync_handle, plist, link) {
1322                 if (index == 0) {
1323                         phdr = pasync_handle->pbuffer;
1324                         hdr_len = pasync_handle->buffer_len;
1325                 } else {
1326                         buf_len = pasync_handle->buffer_len;
1327                         if (!num_buf) {
1328                                 pfirst_buffer = pasync_handle->pbuffer;
1329                                 num_buf++;
1330                         }
1331                         memcpy(pfirst_buffer + offset,
1332                                pasync_handle->pbuffer, buf_len);
1333                         offset = buf_len;
1334                 }
1335                 index++;
1336         }
1337
1338         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1339                                            (beiscsi_conn->beiscsi_conn_cid -
1340                                             phba->fw_config.iscsi_cid_start),
1341                                             phdr, hdr_len, pfirst_buffer,
1342                                             buf_len);
1343
1344         if (status == 0)
1345                 hwi_free_async_msg(phba, cri);
1346         return 0;
1347 }
1348
1349 static unsigned int
1350 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1351                      struct beiscsi_hba *phba,
1352                      struct async_pdu_handle *pasync_handle)
1353 {
1354         struct hwi_async_pdu_context *pasync_ctx;
1355         struct hwi_controller *phwi_ctrlr;
1356         unsigned int bytes_needed = 0, status = 0;
1357         unsigned short cri = pasync_handle->cri;
1358         struct pdu_base *ppdu;
1359
1360         phwi_ctrlr = phba->phwi_ctrlr;
1361         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1362
1363         list_del(&pasync_handle->link);
1364         if (pasync_handle->is_header) {
1365                 pasync_ctx->async_header.busy_entries--;
1366                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1367                         hwi_free_async_msg(phba, cri);
1368                         BUG();
1369                 }
1370
1371                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1372                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1373                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1374                                 (unsigned short)pasync_handle->buffer_len;
1375                 list_add_tail(&pasync_handle->link,
1376                               &pasync_ctx->async_entry[cri].wait_queue.list);
1377
1378                 ppdu = pasync_handle->pbuffer;
1379                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1380                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1381                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1382                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1383                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1384
1385                 if (status == 0) {
1386                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1387                             bytes_needed;
1388
1389                         if (bytes_needed == 0)
1390                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1391                                                            pasync_ctx, cri);
1392                 }
1393         } else {
1394                 pasync_ctx->async_data.busy_entries--;
1395                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1396                         list_add_tail(&pasync_handle->link,
1397                                       &pasync_ctx->async_entry[cri].wait_queue.
1398                                       list);
1399                         pasync_ctx->async_entry[cri].wait_queue.
1400                                 bytes_received +=
1401                                 (unsigned short)pasync_handle->buffer_len;
1402
1403                         if (pasync_ctx->async_entry[cri].wait_queue.
1404                             bytes_received >=
1405                             pasync_ctx->async_entry[cri].wait_queue.
1406                             bytes_needed)
1407                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1408                                                            pasync_ctx, cri);
1409                 }
1410         }
1411         return status;
1412 }
1413
1414 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1415                                          struct beiscsi_hba *phba,
1416                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1417 {
1418         struct hwi_controller *phwi_ctrlr;
1419         struct hwi_async_pdu_context *pasync_ctx;
1420         struct async_pdu_handle *pasync_handle = NULL;
1421         unsigned int cq_index = -1;
1422
1423         phwi_ctrlr = phba->phwi_ctrlr;
1424         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1425         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1426                                              pdpdu_cqe, &cq_index);
1427
1428         if (pasync_handle->consumed == 0)
1429                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1430                                            cq_index);
1431         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1432         hwi_post_async_buffers(phba, pasync_handle->is_header);
1433 }
1434
1435
1436 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1437 {
1438         struct be_queue_info *cq;
1439         struct sol_cqe *sol;
1440         struct dmsg_cqe *dmsg;
1441         unsigned int num_processed = 0;
1442         unsigned int tot_nump = 0;
1443         struct beiscsi_conn *beiscsi_conn;
1444         struct sgl_handle *psgl_handle = NULL;
1445         struct beiscsi_endpoint *beiscsi_ep;
1446         struct iscsi_endpoint *ep;
1447         struct beiscsi_hba *phba;
1448
1449         cq = pbe_eq->cq;
1450         sol = queue_tail_node(cq);
1451         phba = pbe_eq->phba;
1452
1453         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1454                CQE_VALID_MASK) {
1455                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1456
1457                 if (ring_mode) {
1458                         psgl_handle = phba->sgl_hndl_array[((sol->
1459                                       dw[offsetof(struct amap_sol_cqe_ring,
1460                                       icd_index) / 32] & SOL_ICD_INDEX_MASK)
1461                                       >> 6)];
1462                         ep = phba->ep_array[psgl_handle->cid];
1463                 } else {
1464                         ep = phba->ep_array[(u32) ((sol->
1465                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1466                                    SOL_CID_MASK) >> 6) -
1467                                    phba->fw_config.iscsi_cid_start];
1468                 }
1469                 beiscsi_ep = ep->dd_data;
1470                 beiscsi_conn = beiscsi_ep->conn;
1471                 if (num_processed >= 32) {
1472                         hwi_ring_cq_db(phba, cq->id,
1473                                         num_processed, 0, 0);
1474                         tot_nump += num_processed;
1475                         num_processed = 0;
1476                 }
1477
1478                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1479                         32] & CQE_CODE_MASK) {
1480                 case SOL_CMD_COMPLETE:
1481                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1482                         break;
1483                 case DRIVERMSG_NOTIFY:
1484                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1485                         dmsg = (struct dmsg_cqe *)sol;
1486                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1487                         break;
1488                 case UNSOL_HDR_NOTIFY:
1489                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1490                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1491                                              (struct i_t_dpdu_cqe *)sol);
1492                         break;
1493                 case UNSOL_DATA_NOTIFY:
1494                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1495                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1496                                              (struct i_t_dpdu_cqe *)sol);
1497                         break;
1498                 case CXN_INVALIDATE_INDEX_NOTIFY:
1499                 case CMD_INVALIDATED_NOTIFY:
1500                 case CXN_INVALIDATE_NOTIFY:
1501                         SE_DEBUG(DBG_LVL_1,
1502                                  "Ignoring CQ Error notification for cmd/cxn"
1503                                  "invalidate\n");
1504                         break;
1505                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1506                 case CMD_KILLED_INVALID_STATSN_RCVD:
1507                 case CMD_KILLED_INVALID_R2T_RCVD:
1508                 case CMD_CXN_KILLED_LUN_INVALID:
1509                 case CMD_CXN_KILLED_ICD_INVALID:
1510                 case CMD_CXN_KILLED_ITT_INVALID:
1511                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1512                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1513                         if (ring_mode) {
1514                                 SE_DEBUG(DBG_LVL_1,
1515                                  "CQ Error notification for cmd.. "
1516                                  "code %d cid 0x%x\n",
1517                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1518                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1519                         } else {
1520                                 SE_DEBUG(DBG_LVL_1,
1521                                  "CQ Error notification for cmd.. "
1522                                  "code %d cid 0x%x\n",
1523                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1524                                  32] & CQE_CODE_MASK,
1525                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1526                                  32] & SOL_CID_MASK));
1527                         }
1528                         break;
1529                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1530                         SE_DEBUG(DBG_LVL_1,
1531                                  "Digest error on def pdu ring, dropping..\n");
1532                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1533                                              (struct i_t_dpdu_cqe *) sol);
1534                         break;
1535                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1536                 case CXN_KILLED_BURST_LEN_MISMATCH:
1537                 case CXN_KILLED_AHS_RCVD:
1538                 case CXN_KILLED_HDR_DIGEST_ERR:
1539                 case CXN_KILLED_UNKNOWN_HDR:
1540                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1541                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1542                 case CXN_KILLED_TIMED_OUT:
1543                 case CXN_KILLED_FIN_RCVD:
1544                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1545                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1546                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1547                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1548                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1549                         if (ring_mode) {
1550                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1551                                  "0x%x...\n",
1552                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1553                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1554                         } else {
1555                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1556                                  "0x%x...\n",
1557                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1558                                  32] & CQE_CODE_MASK,
1559                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1560                                  32] & CQE_CID_MASK));
1561                         }
1562                         iscsi_conn_failure(beiscsi_conn->conn,
1563                                            ISCSI_ERR_CONN_FAILED);
1564                         break;
1565                 case CXN_KILLED_RST_SENT:
1566                 case CXN_KILLED_RST_RCVD:
1567                         if (ring_mode) {
1568                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1569                                 "received/sent on CID 0x%x...\n",
1570                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1571                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1572                         } else {
1573                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1574                                 "received/sent on CID 0x%x...\n",
1575                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1576                                  32] & CQE_CODE_MASK,
1577                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1578                                  32] & CQE_CID_MASK));
1579                         }
1580                         iscsi_conn_failure(beiscsi_conn->conn,
1581                                            ISCSI_ERR_CONN_FAILED);
1582                         break;
1583                 default:
1584                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1585                                  "received on CID 0x%x...\n",
1586                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1587                                  32] & CQE_CODE_MASK,
1588                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1589                                  32] & CQE_CID_MASK));
1590                         break;
1591                 }
1592
1593                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1594                 queue_tail_inc(cq);
1595                 sol = queue_tail_node(cq);
1596                 num_processed++;
1597         }
1598
1599         if (num_processed > 0) {
1600                 tot_nump += num_processed;
1601                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1602         }
1603         return tot_nump;
1604 }
1605
1606 static void beiscsi_process_all_cqs(struct work_struct *work)
1607 {
1608         unsigned long flags;
1609         struct hwi_controller *phwi_ctrlr;
1610         struct hwi_context_memory *phwi_context;
1611         struct be_eq_obj *pbe_eq;
1612         struct beiscsi_hba *phba =
1613             container_of(work, struct beiscsi_hba, work_cqs);
1614
1615         phwi_ctrlr = phba->phwi_ctrlr;
1616         phwi_context = phwi_ctrlr->phwi_ctxt;
1617         if (phba->msix_enabled)
1618                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1619         else
1620                 pbe_eq = &phwi_context->be_eq[0];
1621
1622         if (phba->todo_mcc_cq) {
1623                 spin_lock_irqsave(&phba->isr_lock, flags);
1624                 phba->todo_mcc_cq = 0;
1625                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1626         }
1627
1628         if (phba->todo_cq) {
1629                 spin_lock_irqsave(&phba->isr_lock, flags);
1630                 phba->todo_cq = 0;
1631                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1632                 beiscsi_process_cq(pbe_eq);
1633         }
1634 }
1635
1636 static int be_iopoll(struct blk_iopoll *iop, int budget)
1637 {
1638         static unsigned int ret;
1639         struct beiscsi_hba *phba;
1640         struct be_eq_obj *pbe_eq;
1641
1642         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1643         ret = beiscsi_process_cq(pbe_eq);
1644         if (ret < budget) {
1645                 phba = pbe_eq->phba;
1646                 blk_iopoll_complete(iop);
1647                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1648                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1649         }
1650         return ret;
1651 }
1652
1653 static void
1654 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1655               unsigned int num_sg, struct beiscsi_io_task *io_task)
1656 {
1657         struct iscsi_sge *psgl;
1658         unsigned short sg_len, index;
1659         unsigned int sge_len = 0;
1660         unsigned long long addr;
1661         struct scatterlist *l_sg;
1662         unsigned int offset;
1663
1664         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1665                                       io_task->bhs_pa.u.a32.address_lo);
1666         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1667                                       io_task->bhs_pa.u.a32.address_hi);
1668
1669         l_sg = sg;
1670         for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1671                 if (index == 0) {
1672                         sg_len = sg_dma_len(sg);
1673                         addr = (u64) sg_dma_address(sg);
1674                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1675                                                         (addr & 0xFFFFFFFF));
1676                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1677                                                         (addr >> 32));
1678                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1679                                                         sg_len);
1680                         sge_len = sg_len;
1681                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1682                                                         1);
1683                 } else {
1684                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1685                                                         0);
1686                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1687                                                         pwrb, sge_len);
1688                         sg_len = sg_dma_len(sg);
1689                         addr = (u64) sg_dma_address(sg);
1690                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1691                                                         (addr & 0xFFFFFFFF));
1692                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1693                                                         (addr >> 32));
1694                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1695                                                         sg_len);
1696                 }
1697         }
1698         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1699         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1700
1701         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1702
1703         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1704                         io_task->bhs_pa.u.a32.address_hi);
1705         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1706                         io_task->bhs_pa.u.a32.address_lo);
1707
1708         if (num_sg == 2)
1709                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1710         sg = l_sg;
1711         psgl++;
1712         psgl++;
1713         offset = 0;
1714         for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1715                 sg_len = sg_dma_len(sg);
1716                 addr = (u64) sg_dma_address(sg);
1717                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1718                                                 (addr & 0xFFFFFFFF));
1719                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1720                                                 (addr >> 32));
1721                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1722                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1723                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1724                 offset += sg_len;
1725         }
1726         psgl--;
1727         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1728 }
1729
1730 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1731 {
1732         struct iscsi_sge *psgl;
1733         unsigned long long addr;
1734         struct beiscsi_io_task *io_task = task->dd_data;
1735         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1736         struct beiscsi_hba *phba = beiscsi_conn->phba;
1737
1738         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1739         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1740                                 io_task->bhs_pa.u.a32.address_lo);
1741         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1742                                 io_task->bhs_pa.u.a32.address_hi);
1743
1744         if (task->data) {
1745                 if (task->data_count) {
1746                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1747                         addr = (u64) pci_map_single(phba->pcidev,
1748                                                     task->data,
1749                                                     task->data_count, 1);
1750                 } else {
1751                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1752                         addr = 0;
1753                 }
1754                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1755                                                 (addr & 0xFFFFFFFF));
1756                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1757                                                 (addr >> 32));
1758                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1759                                                 task->data_count);
1760
1761                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1762         } else {
1763                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1764                 addr = 0;
1765         }
1766
1767         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1768
1769         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1770
1771         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1772                       io_task->bhs_pa.u.a32.address_hi);
1773         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1774                       io_task->bhs_pa.u.a32.address_lo);
1775         if (task->data) {
1776                 psgl++;
1777                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1778                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1779                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1780                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1781                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1782                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1783
1784                 psgl++;
1785                 if (task->data) {
1786                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1787                                                 (addr & 0xFFFFFFFF));
1788                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1789                                                 (addr >> 32));
1790                 }
1791                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1792         }
1793         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1794 }
1795
1796 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1797 {
1798         unsigned int num_cq_pages, num_async_pdu_buf_pages;
1799         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1800         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1801
1802         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1803                                       sizeof(struct sol_cqe));
1804         num_async_pdu_buf_pages =
1805                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1806                                        phba->params.defpdu_hdr_sz);
1807         num_async_pdu_buf_sgl_pages =
1808                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1809                                        sizeof(struct phys_addr));
1810         num_async_pdu_data_pages =
1811                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1812                                        phba->params.defpdu_data_sz);
1813         num_async_pdu_data_sgl_pages =
1814                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1815                                        sizeof(struct phys_addr));
1816
1817         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1818
1819         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1820                                                  BE_ISCSI_PDU_HEADER_SIZE;
1821         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1822                                             sizeof(struct hwi_context_memory);
1823
1824
1825         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1826             * (phba->params.wrbs_per_cxn)
1827             * phba->params.cxns_per_ctrl;
1828         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1829                                  (phba->params.wrbs_per_cxn);
1830         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1831                                 phba->params.cxns_per_ctrl);
1832
1833         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1834                 phba->params.icds_per_ctrl;
1835         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1836                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1837
1838         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1839                 num_async_pdu_buf_pages * PAGE_SIZE;
1840         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1841                 num_async_pdu_data_pages * PAGE_SIZE;
1842         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1843                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1844         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1845                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1846         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1847                 phba->params.asyncpdus_per_ctrl *
1848                 sizeof(struct async_pdu_handle);
1849         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1850                 phba->params.asyncpdus_per_ctrl *
1851                 sizeof(struct async_pdu_handle);
1852         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1853                 sizeof(struct hwi_async_pdu_context) +
1854                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1855 }
1856
1857 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1858 {
1859         struct be_mem_descriptor *mem_descr;
1860         dma_addr_t bus_add;
1861         struct mem_array *mem_arr, *mem_arr_orig;
1862         unsigned int i, j, alloc_size, curr_alloc_size;
1863
1864         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1865         if (!phba->phwi_ctrlr)
1866                 return -ENOMEM;
1867
1868         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1869                                  GFP_KERNEL);
1870         if (!phba->init_mem) {
1871                 kfree(phba->phwi_ctrlr);
1872                 return -ENOMEM;
1873         }
1874
1875         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1876                                GFP_KERNEL);
1877         if (!mem_arr_orig) {
1878                 kfree(phba->init_mem);
1879                 kfree(phba->phwi_ctrlr);
1880                 return -ENOMEM;
1881         }
1882
1883         mem_descr = phba->init_mem;
1884         for (i = 0; i < SE_MEM_MAX; i++) {
1885                 j = 0;
1886                 mem_arr = mem_arr_orig;
1887                 alloc_size = phba->mem_req[i];
1888                 memset(mem_arr, 0, sizeof(struct mem_array) *
1889                        BEISCSI_MAX_FRAGS_INIT);
1890                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1891                 do {
1892                         mem_arr->virtual_address = pci_alloc_consistent(
1893                                                         phba->pcidev,
1894                                                         curr_alloc_size,
1895                                                         &bus_add);
1896                         if (!mem_arr->virtual_address) {
1897                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1898                                         goto free_mem;
1899                                 if (curr_alloc_size -
1900                                         rounddown_pow_of_two(curr_alloc_size))
1901                                         curr_alloc_size = rounddown_pow_of_two
1902                                                              (curr_alloc_size);
1903                                 else
1904                                         curr_alloc_size = curr_alloc_size / 2;
1905                         } else {
1906                                 mem_arr->bus_address.u.
1907                                     a64.address = (__u64) bus_add;
1908                                 mem_arr->size = curr_alloc_size;
1909                                 alloc_size -= curr_alloc_size;
1910                                 curr_alloc_size = min(be_max_phys_size *
1911                                                       1024, alloc_size);
1912                                 j++;
1913                                 mem_arr++;
1914                         }
1915                 } while (alloc_size);
1916                 mem_descr->num_elements = j;
1917                 mem_descr->size_in_bytes = phba->mem_req[i];
1918                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1919                                                GFP_KERNEL);
1920                 if (!mem_descr->mem_array)
1921                         goto free_mem;
1922
1923                 memcpy(mem_descr->mem_array, mem_arr_orig,
1924                        sizeof(struct mem_array) * j);
1925                 mem_descr++;
1926         }
1927         kfree(mem_arr_orig);
1928         return 0;
1929 free_mem:
1930         mem_descr->num_elements = j;
1931         while ((i) || (j)) {
1932                 for (j = mem_descr->num_elements; j > 0; j--) {
1933                         pci_free_consistent(phba->pcidev,
1934                                             mem_descr->mem_array[j - 1].size,
1935                                             mem_descr->mem_array[j - 1].
1936                                             virtual_address,
1937                                             mem_descr->mem_array[j - 1].
1938                                             bus_address.u.a64.address);
1939                 }
1940                 if (i) {
1941                         i--;
1942                         kfree(mem_descr->mem_array);
1943                         mem_descr--;
1944                 }
1945         }
1946         kfree(mem_arr_orig);
1947         kfree(phba->init_mem);
1948         kfree(phba->phwi_ctrlr);
1949         return -ENOMEM;
1950 }
1951
1952 static int beiscsi_get_memory(struct beiscsi_hba *phba)
1953 {
1954         beiscsi_find_mem_req(phba);
1955         return beiscsi_alloc_mem(phba);
1956 }
1957
1958 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1959 {
1960         struct pdu_data_out *pdata_out;
1961         struct pdu_nop_out *pnop_out;
1962         struct be_mem_descriptor *mem_descr;
1963
1964         mem_descr = phba->init_mem;
1965         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1966         pdata_out =
1967             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1968         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1969
1970         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1971                       IIOC_SCSI_DATA);
1972
1973         pnop_out =
1974             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1975                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1976
1977         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1978         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1979         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1980         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1981 }
1982
1983 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1984 {
1985         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1986         struct wrb_handle *pwrb_handle;
1987         struct hwi_controller *phwi_ctrlr;
1988         struct hwi_wrb_context *pwrb_context;
1989         struct iscsi_wrb *pwrb;
1990         unsigned int num_cxn_wrbh;
1991         unsigned int num_cxn_wrb, j, idx, index;
1992
1993         mem_descr_wrbh = phba->init_mem;
1994         mem_descr_wrbh += HWI_MEM_WRBH;
1995
1996         mem_descr_wrb = phba->init_mem;
1997         mem_descr_wrb += HWI_MEM_WRB;
1998
1999         idx = 0;
2000         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2001         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2002                         ((sizeof(struct wrb_handle)) *
2003                          phba->params.wrbs_per_cxn));
2004         phwi_ctrlr = phba->phwi_ctrlr;
2005
2006         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2007                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2008                 pwrb_context->pwrb_handle_base =
2009                                 kzalloc(sizeof(struct wrb_handle *) *
2010                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2011                 pwrb_context->pwrb_handle_basestd =
2012                                 kzalloc(sizeof(struct wrb_handle *) *
2013                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2014                 if (num_cxn_wrbh) {
2015                         pwrb_context->alloc_index = 0;
2016                         pwrb_context->wrb_handles_available = 0;
2017                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2018                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2019                                 pwrb_context->pwrb_handle_basestd[j] =
2020                                                                 pwrb_handle;
2021                                 pwrb_context->wrb_handles_available++;
2022                                 pwrb_handle->wrb_index = j;
2023                                 pwrb_handle++;
2024                         }
2025                         pwrb_context->free_index = 0;
2026                         num_cxn_wrbh--;
2027                 } else {
2028                         idx++;
2029                         pwrb_handle =
2030                             mem_descr_wrbh->mem_array[idx].virtual_address;
2031                         num_cxn_wrbh =
2032                             ((mem_descr_wrbh->mem_array[idx].size) /
2033                              ((sizeof(struct wrb_handle)) *
2034                               phba->params.wrbs_per_cxn));
2035                         pwrb_context->alloc_index = 0;
2036                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2037                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2038                                 pwrb_context->pwrb_handle_basestd[j] =
2039                                     pwrb_handle;
2040                                 pwrb_context->wrb_handles_available++;
2041                                 pwrb_handle->wrb_index = j;
2042                                 pwrb_handle++;
2043                         }
2044                         pwrb_context->free_index = 0;
2045                         num_cxn_wrbh--;
2046                 }
2047         }
2048         idx = 0;
2049         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2050         num_cxn_wrb =
2051             ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
2052              phba->params.wrbs_per_cxn);
2053
2054         for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
2055                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2056                 if (num_cxn_wrb) {
2057                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2058                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2059                                 pwrb_handle->pwrb = pwrb;
2060                                 pwrb++;
2061                         }
2062                         num_cxn_wrb--;
2063                 } else {
2064                         idx++;
2065                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2066                         num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
2067                                         (sizeof(struct iscsi_wrb)) *
2068                                         phba->params.wrbs_per_cxn);
2069                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2070                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2071                                 pwrb_handle->pwrb = pwrb;
2072                                 pwrb++;
2073                         }
2074                         num_cxn_wrb--;
2075                 }
2076         }
2077 }
2078
2079 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2080 {
2081         struct hwi_controller *phwi_ctrlr;
2082         struct hba_parameters *p = &phba->params;
2083         struct hwi_async_pdu_context *pasync_ctx;
2084         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2085         unsigned int index;
2086         struct be_mem_descriptor *mem_descr;
2087
2088         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2089         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2090
2091         phwi_ctrlr = phba->phwi_ctrlr;
2092         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2093                                 mem_descr->mem_array[0].virtual_address;
2094         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2095         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2096
2097         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2098         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2099         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2100         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2101
2102         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2103         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2104         if (mem_descr->mem_array[0].virtual_address) {
2105                 SE_DEBUG(DBG_LVL_8,
2106                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2107                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2108         } else
2109                 shost_printk(KERN_WARNING, phba->shost,
2110                              "No Virtual address \n");
2111
2112         pasync_ctx->async_header.va_base =
2113                         mem_descr->mem_array[0].virtual_address;
2114
2115         pasync_ctx->async_header.pa_base.u.a64.address =
2116                         mem_descr->mem_array[0].bus_address.u.a64.address;
2117
2118         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2119         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2120         if (mem_descr->mem_array[0].virtual_address) {
2121                 SE_DEBUG(DBG_LVL_8,
2122                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2123                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2124         } else
2125                 shost_printk(KERN_WARNING, phba->shost,
2126                             "No Virtual address \n");
2127         pasync_ctx->async_header.ring_base =
2128                         mem_descr->mem_array[0].virtual_address;
2129
2130         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2131         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2132         if (mem_descr->mem_array[0].virtual_address) {
2133                 SE_DEBUG(DBG_LVL_8,
2134                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2135                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2136         } else
2137                 shost_printk(KERN_WARNING, phba->shost,
2138                             "No Virtual address \n");
2139
2140         pasync_ctx->async_header.handle_base =
2141                         mem_descr->mem_array[0].virtual_address;
2142         pasync_ctx->async_header.writables = 0;
2143         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2144
2145         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2146         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2147         if (mem_descr->mem_array[0].virtual_address) {
2148                 SE_DEBUG(DBG_LVL_8,
2149                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2150                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2151         } else
2152                 shost_printk(KERN_WARNING, phba->shost,
2153                             "No Virtual address \n");
2154         pasync_ctx->async_data.va_base =
2155                         mem_descr->mem_array[0].virtual_address;
2156         pasync_ctx->async_data.pa_base.u.a64.address =
2157                         mem_descr->mem_array[0].bus_address.u.a64.address;
2158
2159         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2160         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2161         if (mem_descr->mem_array[0].virtual_address) {
2162                 SE_DEBUG(DBG_LVL_8,
2163                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2164                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2165         } else
2166                 shost_printk(KERN_WARNING, phba->shost,
2167                              "No Virtual address \n");
2168
2169         pasync_ctx->async_data.ring_base =
2170                         mem_descr->mem_array[0].virtual_address;
2171
2172         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2173         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2174         if (!mem_descr->mem_array[0].virtual_address)
2175                 shost_printk(KERN_WARNING, phba->shost,
2176                             "No Virtual address \n");
2177
2178         pasync_ctx->async_data.handle_base =
2179                         mem_descr->mem_array[0].virtual_address;
2180         pasync_ctx->async_data.writables = 0;
2181         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2182
2183         pasync_header_h =
2184                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2185         pasync_data_h =
2186                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2187
2188         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2189                 pasync_header_h->cri = -1;
2190                 pasync_header_h->index = (char)index;
2191                 INIT_LIST_HEAD(&pasync_header_h->link);
2192                 pasync_header_h->pbuffer =
2193                         (void *)((unsigned long)
2194                         (pasync_ctx->async_header.va_base) +
2195                         (p->defpdu_hdr_sz * index));
2196
2197                 pasync_header_h->pa.u.a64.address =
2198                         pasync_ctx->async_header.pa_base.u.a64.address +
2199                         (p->defpdu_hdr_sz * index);
2200
2201                 list_add_tail(&pasync_header_h->link,
2202                                 &pasync_ctx->async_header.free_list);
2203                 pasync_header_h++;
2204                 pasync_ctx->async_header.free_entries++;
2205                 pasync_ctx->async_header.writables++;
2206
2207                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2208                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2209                                header_busy_list);
2210                 pasync_data_h->cri = -1;
2211                 pasync_data_h->index = (char)index;
2212                 INIT_LIST_HEAD(&pasync_data_h->link);
2213                 pasync_data_h->pbuffer =
2214                         (void *)((unsigned long)
2215                         (pasync_ctx->async_data.va_base) +
2216                         (p->defpdu_data_sz * index));
2217
2218                 pasync_data_h->pa.u.a64.address =
2219                     pasync_ctx->async_data.pa_base.u.a64.address +
2220                     (p->defpdu_data_sz * index);
2221
2222                 list_add_tail(&pasync_data_h->link,
2223                               &pasync_ctx->async_data.free_list);
2224                 pasync_data_h++;
2225                 pasync_ctx->async_data.free_entries++;
2226                 pasync_ctx->async_data.writables++;
2227
2228                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2229         }
2230
2231         pasync_ctx->async_header.host_write_ptr = 0;
2232         pasync_ctx->async_header.ep_read_ptr = -1;
2233         pasync_ctx->async_data.host_write_ptr = 0;
2234         pasync_ctx->async_data.ep_read_ptr = -1;
2235 }
2236
2237 static int
2238 be_sgl_create_contiguous(void *virtual_address,
2239                          u64 physical_address, u32 length,
2240                          struct be_dma_mem *sgl)
2241 {
2242         WARN_ON(!virtual_address);
2243         WARN_ON(!physical_address);
2244         WARN_ON(!length > 0);
2245         WARN_ON(!sgl);
2246
2247         sgl->va = virtual_address;
2248         sgl->dma = physical_address;
2249         sgl->size = length;
2250
2251         return 0;
2252 }
2253
2254 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2255 {
2256         memset(sgl, 0, sizeof(*sgl));
2257 }
2258
2259 static void
2260 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2261                      struct mem_array *pmem, struct be_dma_mem *sgl)
2262 {
2263         if (sgl->va)
2264                 be_sgl_destroy_contiguous(sgl);
2265
2266         be_sgl_create_contiguous(pmem->virtual_address,
2267                                  pmem->bus_address.u.a64.address,
2268                                  pmem->size, sgl);
2269 }
2270
2271 static void
2272 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2273                            struct mem_array *pmem, struct be_dma_mem *sgl)
2274 {
2275         if (sgl->va)
2276                 be_sgl_destroy_contiguous(sgl);
2277
2278         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2279                                  pmem->bus_address.u.a64.address,
2280                                  pmem->size, sgl);
2281 }
2282
2283 static int be_fill_queue(struct be_queue_info *q,
2284                 u16 len, u16 entry_size, void *vaddress)
2285 {
2286         struct be_dma_mem *mem = &q->dma_mem;
2287
2288         memset(q, 0, sizeof(*q));
2289         q->len = len;
2290         q->entry_size = entry_size;
2291         mem->size = len * entry_size;
2292         mem->va = vaddress;
2293         if (!mem->va)
2294                 return -ENOMEM;
2295         memset(mem->va, 0, mem->size);
2296         return 0;
2297 }
2298
2299 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2300                              struct hwi_context_memory *phwi_context)
2301 {
2302         unsigned int i, num_eq_pages;
2303         int ret, eq_for_mcc;
2304         struct be_queue_info *eq;
2305         struct be_dma_mem *mem;
2306         void *eq_vaddress;
2307         dma_addr_t paddr;
2308
2309         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2310                                       sizeof(struct be_eq_entry));
2311
2312         if (phba->msix_enabled)
2313                 eq_for_mcc = 1;
2314         else
2315                 eq_for_mcc = 0;
2316         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2317                 eq = &phwi_context->be_eq[i].q;
2318                 mem = &eq->dma_mem;
2319                 phwi_context->be_eq[i].phba = phba;
2320                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2321                                                      num_eq_pages * PAGE_SIZE,
2322                                                      &paddr);
2323                 if (!eq_vaddress)
2324                         goto create_eq_error;
2325
2326                 mem->va = eq_vaddress;
2327                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2328                                     sizeof(struct be_eq_entry), eq_vaddress);
2329                 if (ret) {
2330                         shost_printk(KERN_ERR, phba->shost,
2331                                      "be_fill_queue Failed for EQ \n");
2332                         goto create_eq_error;
2333                 }
2334
2335                 mem->dma = paddr;
2336                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2337                                             phwi_context->cur_eqd);
2338                 if (ret) {
2339                         shost_printk(KERN_ERR, phba->shost,
2340                                      "beiscsi_cmd_eq_create"
2341                                      "Failedfor EQ \n");
2342                         goto create_eq_error;
2343                 }
2344                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2345         }
2346         return 0;
2347 create_eq_error:
2348         for (i = 0; i < (phba->num_cpus + 1); i++) {
2349                 eq = &phwi_context->be_eq[i].q;
2350                 mem = &eq->dma_mem;
2351                 if (mem->va)
2352                         pci_free_consistent(phba->pcidev, num_eq_pages
2353                                             * PAGE_SIZE,
2354                                             mem->va, mem->dma);
2355         }
2356         return ret;
2357 }
2358
2359 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2360                              struct hwi_context_memory *phwi_context)
2361 {
2362         unsigned int i, num_cq_pages;
2363         int ret;
2364         struct be_queue_info *cq, *eq;
2365         struct be_dma_mem *mem;
2366         struct be_eq_obj *pbe_eq;
2367         void *cq_vaddress;
2368         dma_addr_t paddr;
2369
2370         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2371                                       sizeof(struct sol_cqe));
2372
2373         for (i = 0; i < phba->num_cpus; i++) {
2374                 cq = &phwi_context->be_cq[i];
2375                 eq = &phwi_context->be_eq[i].q;
2376                 pbe_eq = &phwi_context->be_eq[i];
2377                 pbe_eq->cq = cq;
2378                 pbe_eq->phba = phba;
2379                 mem = &cq->dma_mem;
2380                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2381                                                      num_cq_pages * PAGE_SIZE,
2382                                                      &paddr);
2383                 if (!cq_vaddress)
2384                         goto create_cq_error;
2385                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2386                                     sizeof(struct sol_cqe), cq_vaddress);
2387                 if (ret) {
2388                         shost_printk(KERN_ERR, phba->shost,
2389                                      "be_fill_queue Failed for ISCSI CQ \n");
2390                         goto create_cq_error;
2391                 }
2392
2393                 mem->dma = paddr;
2394                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2395                                             false, 0);
2396                 if (ret) {
2397                         shost_printk(KERN_ERR, phba->shost,
2398                                      "beiscsi_cmd_eq_create"
2399                                      "Failed for ISCSI CQ \n");
2400                         goto create_cq_error;
2401                 }
2402                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2403                                                  cq->id, eq->id);
2404                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2405         }
2406         return 0;
2407
2408 create_cq_error:
2409         for (i = 0; i < phba->num_cpus; i++) {
2410                 cq = &phwi_context->be_cq[i];
2411                 mem = &cq->dma_mem;
2412                 if (mem->va)
2413                         pci_free_consistent(phba->pcidev, num_cq_pages
2414                                             * PAGE_SIZE,
2415                                             mem->va, mem->dma);
2416         }
2417         return ret;
2418
2419 }
2420
2421 static int
2422 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2423                        struct hwi_context_memory *phwi_context,
2424                        struct hwi_controller *phwi_ctrlr,
2425                        unsigned int def_pdu_ring_sz)
2426 {
2427         unsigned int idx;
2428         int ret;
2429         struct be_queue_info *dq, *cq;
2430         struct be_dma_mem *mem;
2431         struct be_mem_descriptor *mem_descr;
2432         void *dq_vaddress;
2433
2434         idx = 0;
2435         dq = &phwi_context->be_def_hdrq;
2436         cq = &phwi_context->be_cq[0];
2437         mem = &dq->dma_mem;
2438         mem_descr = phba->init_mem;
2439         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2440         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2441         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2442                             sizeof(struct phys_addr),
2443                             sizeof(struct phys_addr), dq_vaddress);
2444         if (ret) {
2445                 shost_printk(KERN_ERR, phba->shost,
2446                              "be_fill_queue Failed for DEF PDU HDR\n");
2447                 return ret;
2448         }
2449         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2450         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2451                                               def_pdu_ring_sz,
2452                                               phba->params.defpdu_hdr_sz);
2453         if (ret) {
2454                 shost_printk(KERN_ERR, phba->shost,
2455                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2456                 return ret;
2457         }
2458         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2459         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2460                  phwi_context->be_def_hdrq.id);
2461         hwi_post_async_buffers(phba, 1);
2462         return 0;
2463 }
2464
2465 static int
2466 beiscsi_create_def_data(struct beiscsi_hba *phba,
2467                         struct hwi_context_memory *phwi_context,
2468                         struct hwi_controller *phwi_ctrlr,
2469                         unsigned int def_pdu_ring_sz)
2470 {
2471         unsigned int idx;
2472         int ret;
2473         struct be_queue_info *dataq, *cq;
2474         struct be_dma_mem *mem;
2475         struct be_mem_descriptor *mem_descr;
2476         void *dq_vaddress;
2477
2478         idx = 0;
2479         dataq = &phwi_context->be_def_dataq;
2480         cq = &phwi_context->be_cq[0];
2481         mem = &dataq->dma_mem;
2482         mem_descr = phba->init_mem;
2483         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2484         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2485         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2486                             sizeof(struct phys_addr),
2487                             sizeof(struct phys_addr), dq_vaddress);
2488         if (ret) {
2489                 shost_printk(KERN_ERR, phba->shost,
2490                              "be_fill_queue Failed for DEF PDU DATA\n");
2491                 return ret;
2492         }
2493         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2494         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2495                                               def_pdu_ring_sz,
2496                                               phba->params.defpdu_data_sz);
2497         if (ret) {
2498                 shost_printk(KERN_ERR, phba->shost,
2499                              "be_cmd_create_default_pdu_queue Failed"
2500                              " for DEF PDU DATA\n");
2501                 return ret;
2502         }
2503         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2504         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2505                  phwi_context->be_def_dataq.id);
2506         hwi_post_async_buffers(phba, 0);
2507         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2508         return 0;
2509 }
2510
2511 static int
2512 beiscsi_post_pages(struct beiscsi_hba *phba)
2513 {
2514         struct be_mem_descriptor *mem_descr;
2515         struct mem_array *pm_arr;
2516         unsigned int page_offset, i;
2517         struct be_dma_mem sgl;
2518         int status;
2519
2520         mem_descr = phba->init_mem;
2521         mem_descr += HWI_MEM_SGE;
2522         pm_arr = mem_descr->mem_array;
2523
2524         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2525                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2526         for (i = 0; i < mem_descr->num_elements; i++) {
2527                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2528                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2529                                                 page_offset,
2530                                                 (pm_arr->size / PAGE_SIZE));
2531                 page_offset += pm_arr->size / PAGE_SIZE;
2532                 if (status != 0) {
2533                         shost_printk(KERN_ERR, phba->shost,
2534                                      "post sgl failed.\n");
2535                         return status;
2536                 }
2537                 pm_arr++;
2538         }
2539         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2540         return 0;
2541 }
2542
2543 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2544 {
2545         struct be_dma_mem *mem = &q->dma_mem;
2546         if (mem->va)
2547                 pci_free_consistent(phba->pcidev, mem->size,
2548                         mem->va, mem->dma);
2549 }
2550
2551 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2552                 u16 len, u16 entry_size)
2553 {
2554         struct be_dma_mem *mem = &q->dma_mem;
2555
2556         memset(q, 0, sizeof(*q));
2557         q->len = len;
2558         q->entry_size = entry_size;
2559         mem->size = len * entry_size;
2560         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2561         if (!mem->va)
2562                 return -1;
2563         memset(mem->va, 0, mem->size);
2564         return 0;
2565 }
2566
2567 static int
2568 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2569                          struct hwi_context_memory *phwi_context,
2570                          struct hwi_controller *phwi_ctrlr)
2571 {
2572         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2573         u64 pa_addr_lo;
2574         unsigned int idx, num, i;
2575         struct mem_array *pwrb_arr;
2576         void *wrb_vaddr;
2577         struct be_dma_mem sgl;
2578         struct be_mem_descriptor *mem_descr;
2579         int status;
2580
2581         idx = 0;
2582         mem_descr = phba->init_mem;
2583         mem_descr += HWI_MEM_WRB;
2584         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2585                            GFP_KERNEL);
2586         if (!pwrb_arr) {
2587                 shost_printk(KERN_ERR, phba->shost,
2588                              "Memory alloc failed in create wrb ring.\n");
2589                 return -ENOMEM;
2590         }
2591         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2592         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2593         num_wrb_rings = mem_descr->mem_array[idx].size /
2594                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2595
2596         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2597                 if (num_wrb_rings) {
2598                         pwrb_arr[num].virtual_address = wrb_vaddr;
2599                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2600                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2601                                             sizeof(struct iscsi_wrb);
2602                         wrb_vaddr += pwrb_arr[num].size;
2603                         pa_addr_lo += pwrb_arr[num].size;
2604                         num_wrb_rings--;
2605                 } else {
2606                         idx++;
2607                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2608                         pa_addr_lo = mem_descr->mem_array[idx].\
2609                                         bus_address.u.a64.address;
2610                         num_wrb_rings = mem_descr->mem_array[idx].size /
2611                                         (phba->params.wrbs_per_cxn *
2612                                         sizeof(struct iscsi_wrb));
2613                         pwrb_arr[num].virtual_address = wrb_vaddr;
2614                         pwrb_arr[num].bus_address.u.a64.address\
2615                                                 = pa_addr_lo;
2616                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2617                                                  sizeof(struct iscsi_wrb);
2618                         wrb_vaddr += pwrb_arr[num].size;
2619                         pa_addr_lo   += pwrb_arr[num].size;
2620                         num_wrb_rings--;
2621                 }
2622         }
2623         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2624                 wrb_mem_index = 0;
2625                 offset = 0;
2626                 size = 0;
2627
2628                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2629                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2630                                             &phwi_context->be_wrbq[i]);
2631                 if (status != 0) {
2632                         shost_printk(KERN_ERR, phba->shost,
2633                                      "wrbq create failed.");
2634                         return status;
2635                 }
2636                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2637                                                                    id;
2638         }
2639         kfree(pwrb_arr);
2640         return 0;
2641 }
2642
2643 static void free_wrb_handles(struct beiscsi_hba *phba)
2644 {
2645         unsigned int index;
2646         struct hwi_controller *phwi_ctrlr;
2647         struct hwi_wrb_context *pwrb_context;
2648
2649         phwi_ctrlr = phba->phwi_ctrlr;
2650         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2651                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2652                 kfree(pwrb_context->pwrb_handle_base);
2653                 kfree(pwrb_context->pwrb_handle_basestd);
2654         }
2655 }
2656
2657 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2658 {
2659         struct be_queue_info *q;
2660         struct be_ctrl_info *ctrl = &phba->ctrl;
2661
2662         q = &phba->ctrl.mcc_obj.q;
2663         if (q->created)
2664                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2665         be_queue_free(phba, q);
2666
2667         q = &phba->ctrl.mcc_obj.cq;
2668         if (q->created)
2669                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2670         be_queue_free(phba, q);
2671 }
2672
2673 static void hwi_cleanup(struct beiscsi_hba *phba)
2674 {
2675         struct be_queue_info *q;
2676         struct be_ctrl_info *ctrl = &phba->ctrl;
2677         struct hwi_controller *phwi_ctrlr;
2678         struct hwi_context_memory *phwi_context;
2679         int i, eq_num;
2680
2681         phwi_ctrlr = phba->phwi_ctrlr;
2682         phwi_context = phwi_ctrlr->phwi_ctxt;
2683         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2684                 q = &phwi_context->be_wrbq[i];
2685                 if (q->created)
2686                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2687         }
2688         free_wrb_handles(phba);
2689
2690         q = &phwi_context->be_def_hdrq;
2691         if (q->created)
2692                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2693
2694         q = &phwi_context->be_def_dataq;
2695         if (q->created)
2696                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2697
2698         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2699
2700         for (i = 0; i < (phba->num_cpus); i++) {
2701                 q = &phwi_context->be_cq[i];
2702                 if (q->created)
2703                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2704         }
2705         if (phba->msix_enabled)
2706                 eq_num = 1;
2707         else
2708                 eq_num = 0;
2709         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2710                 q = &phwi_context->be_eq[i].q;
2711                 if (q->created)
2712                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2713         }
2714         be_mcc_queues_destroy(phba);
2715 }
2716
2717 static int be_mcc_queues_create(struct beiscsi_hba *phba,
2718                                 struct hwi_context_memory *phwi_context)
2719 {
2720         struct be_queue_info *q, *cq;
2721         struct be_ctrl_info *ctrl = &phba->ctrl;
2722
2723         /* Alloc MCC compl queue */
2724         cq = &phba->ctrl.mcc_obj.cq;
2725         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2726                         sizeof(struct be_mcc_compl)))
2727                 goto err;
2728         /* Ask BE to create MCC compl queue; */
2729         if (phba->msix_enabled) {
2730                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2731                                          [phba->num_cpus].q, false, true, 0))
2732                 goto mcc_cq_free;
2733         } else {
2734                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2735                                           false, true, 0))
2736                 goto mcc_cq_free;
2737         }
2738
2739         /* Alloc MCC queue */
2740         q = &phba->ctrl.mcc_obj.q;
2741         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2742                 goto mcc_cq_destroy;
2743
2744         /* Ask BE to create MCC queue */
2745         if (beiscsi_cmd_mccq_create(phba, q, cq))
2746                 goto mcc_q_free;
2747
2748         return 0;
2749
2750 mcc_q_free:
2751         be_queue_free(phba, q);
2752 mcc_cq_destroy:
2753         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2754 mcc_cq_free:
2755         be_queue_free(phba, cq);
2756 err:
2757         return -1;
2758 }
2759
2760 static int find_num_cpus(void)
2761 {
2762         int  num_cpus = 0;
2763
2764         num_cpus = num_online_cpus();
2765         if (num_cpus >= MAX_CPUS)
2766                 num_cpus = MAX_CPUS - 1;
2767
2768         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2769         return num_cpus;
2770 }
2771
2772 static int hwi_init_port(struct beiscsi_hba *phba)
2773 {
2774         struct hwi_controller *phwi_ctrlr;
2775         struct hwi_context_memory *phwi_context;
2776         unsigned int def_pdu_ring_sz;
2777         struct be_ctrl_info *ctrl = &phba->ctrl;
2778         int status;
2779
2780         def_pdu_ring_sz =
2781                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2782         phwi_ctrlr = phba->phwi_ctrlr;
2783         phwi_context = phwi_ctrlr->phwi_ctxt;
2784         phwi_context->max_eqd = 0;
2785         phwi_context->min_eqd = 0;
2786         phwi_context->cur_eqd = 64;
2787         be_cmd_fw_initialize(&phba->ctrl);
2788
2789         status = beiscsi_create_eqs(phba, phwi_context);
2790         if (status != 0) {
2791                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2792                 goto error;
2793         }
2794
2795         status = be_mcc_queues_create(phba, phwi_context);
2796         if (status != 0)
2797                 goto error;
2798
2799         status = mgmt_check_supported_fw(ctrl, phba);
2800         if (status != 0) {
2801                 shost_printk(KERN_ERR, phba->shost,
2802                              "Unsupported fw version \n");
2803                 goto error;
2804         }
2805
2806         if (phba->fw_config.iscsi_features == 0x1)
2807                 ring_mode = 1;
2808         else
2809                 ring_mode = 0;
2810
2811         status = beiscsi_create_cqs(phba, phwi_context);
2812         if (status != 0) {
2813                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2814                 goto error;
2815         }
2816
2817         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2818                                         def_pdu_ring_sz);
2819         if (status != 0) {
2820                 shost_printk(KERN_ERR, phba->shost,
2821                              "Default Header not created\n");
2822                 goto error;
2823         }
2824
2825         status = beiscsi_create_def_data(phba, phwi_context,
2826                                          phwi_ctrlr, def_pdu_ring_sz);
2827         if (status != 0) {
2828                 shost_printk(KERN_ERR, phba->shost,
2829                              "Default Data not created\n");
2830                 goto error;
2831         }
2832
2833         status = beiscsi_post_pages(phba);
2834         if (status != 0) {
2835                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2836                 goto error;
2837         }
2838
2839         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2840         if (status != 0) {
2841                 shost_printk(KERN_ERR, phba->shost,
2842                              "WRB Rings not created\n");
2843                 goto error;
2844         }
2845
2846         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2847         return 0;
2848
2849 error:
2850         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2851         hwi_cleanup(phba);
2852         return -ENOMEM;
2853 }
2854
2855 static int hwi_init_controller(struct beiscsi_hba *phba)
2856 {
2857         struct hwi_controller *phwi_ctrlr;
2858
2859         phwi_ctrlr = phba->phwi_ctrlr;
2860         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2861                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2862                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2863                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2864                          phwi_ctrlr->phwi_ctxt);
2865         } else {
2866                 shost_printk(KERN_ERR, phba->shost,
2867                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2868                              "Failing to load\n");
2869                 return -ENOMEM;
2870         }
2871
2872         iscsi_init_global_templates(phba);
2873         beiscsi_init_wrb_handle(phba);
2874         hwi_init_async_pdu_ctx(phba);
2875         if (hwi_init_port(phba) != 0) {
2876                 shost_printk(KERN_ERR, phba->shost,
2877                              "hwi_init_controller failed\n");
2878                 return -ENOMEM;
2879         }
2880         return 0;
2881 }
2882
2883 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2884 {
2885         struct be_mem_descriptor *mem_descr;
2886         int i, j;
2887
2888         mem_descr = phba->init_mem;
2889         i = 0;
2890         j = 0;
2891         for (i = 0; i < SE_MEM_MAX; i++) {
2892                 for (j = mem_descr->num_elements; j > 0; j--) {
2893                         pci_free_consistent(phba->pcidev,
2894                           mem_descr->mem_array[j - 1].size,
2895                           mem_descr->mem_array[j - 1].virtual_address,
2896                           mem_descr->mem_array[j - 1].bus_address.
2897                                 u.a64.address);
2898                 }
2899                 kfree(mem_descr->mem_array);
2900                 mem_descr++;
2901         }
2902         kfree(phba->init_mem);
2903         kfree(phba->phwi_ctrlr);
2904 }
2905
2906 static int beiscsi_init_controller(struct beiscsi_hba *phba)
2907 {
2908         int ret = -ENOMEM;
2909
2910         ret = beiscsi_get_memory(phba);
2911         if (ret < 0) {
2912                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2913                              "Failed in beiscsi_alloc_memory \n");
2914                 return ret;
2915         }
2916
2917         ret = hwi_init_controller(phba);
2918         if (ret)
2919                 goto free_init;
2920         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2921         return 0;
2922
2923 free_init:
2924         beiscsi_free_mem(phba);
2925         return -ENOMEM;
2926 }
2927
2928 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2929 {
2930         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2931         struct sgl_handle *psgl_handle;
2932         struct iscsi_sge *pfrag;
2933         unsigned int arr_index, i, idx;
2934
2935         phba->io_sgl_hndl_avbl = 0;
2936         phba->eh_sgl_hndl_avbl = 0;
2937
2938         if (ring_mode) {
2939                 phba->sgl_hndl_array = kzalloc(sizeof(struct sgl_handle *) *
2940                                               phba->params.icds_per_ctrl,
2941                                                  GFP_KERNEL);
2942                 if (!phba->sgl_hndl_array) {
2943                         shost_printk(KERN_ERR, phba->shost,
2944                              "Mem Alloc Failed. Failing to load\n");
2945                         return -ENOMEM;
2946                 }
2947         }
2948
2949         mem_descr_sglh = phba->init_mem;
2950         mem_descr_sglh += HWI_MEM_SGLH;
2951         if (1 == mem_descr_sglh->num_elements) {
2952                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2953                                                  phba->params.ios_per_ctrl,
2954                                                  GFP_KERNEL);
2955                 if (!phba->io_sgl_hndl_base) {
2956                         if (ring_mode)
2957                                 kfree(phba->sgl_hndl_array);
2958                         shost_printk(KERN_ERR, phba->shost,
2959                                      "Mem Alloc Failed. Failing to load\n");
2960                         return -ENOMEM;
2961                 }
2962                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2963                                                  (phba->params.icds_per_ctrl -
2964                                                  phba->params.ios_per_ctrl),
2965                                                  GFP_KERNEL);
2966                 if (!phba->eh_sgl_hndl_base) {
2967                         kfree(phba->io_sgl_hndl_base);
2968                         shost_printk(KERN_ERR, phba->shost,
2969                                      "Mem Alloc Failed. Failing to load\n");
2970                         return -ENOMEM;
2971                 }
2972         } else {
2973                 shost_printk(KERN_ERR, phba->shost,
2974                              "HWI_MEM_SGLH is more than one element."
2975                              "Failing to load\n");
2976                 return -ENOMEM;
2977         }
2978
2979         arr_index = 0;
2980         idx = 0;
2981         while (idx < mem_descr_sglh->num_elements) {
2982                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2983
2984                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2985                       sizeof(struct sgl_handle)); i++) {
2986                         if (arr_index < phba->params.ios_per_ctrl) {
2987                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2988                                 phba->io_sgl_hndl_avbl++;
2989                                 arr_index++;
2990                         } else {
2991                                 phba->eh_sgl_hndl_base[arr_index -
2992                                         phba->params.ios_per_ctrl] =
2993                                                                 psgl_handle;
2994                                 arr_index++;
2995                                 phba->eh_sgl_hndl_avbl++;
2996                         }
2997                         psgl_handle++;
2998                 }
2999                 idx++;
3000         }
3001         SE_DEBUG(DBG_LVL_8,
3002                  "phba->io_sgl_hndl_avbl=%d"
3003                  "phba->eh_sgl_hndl_avbl=%d \n",
3004                  phba->io_sgl_hndl_avbl,
3005                  phba->eh_sgl_hndl_avbl);
3006         mem_descr_sg = phba->init_mem;
3007         mem_descr_sg += HWI_MEM_SGE;
3008         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
3009                  mem_descr_sg->num_elements);
3010         arr_index = 0;
3011         idx = 0;
3012         while (idx < mem_descr_sg->num_elements) {
3013                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3014
3015                 for (i = 0;
3016                      i < (mem_descr_sg->mem_array[idx].size) /
3017                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3018                      i++) {
3019                         if (arr_index < phba->params.ios_per_ctrl)
3020                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3021                         else
3022                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3023                                                 phba->params.ios_per_ctrl];
3024                         psgl_handle->pfrag = pfrag;
3025                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3026                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3027                         pfrag += phba->params.num_sge_per_io;
3028                         psgl_handle->sgl_index =
3029                                 phba->fw_config.iscsi_icd_start + arr_index++;
3030                 }
3031                 idx++;
3032         }
3033         phba->io_sgl_free_index = 0;
3034         phba->io_sgl_alloc_index = 0;
3035         phba->eh_sgl_free_index = 0;
3036         phba->eh_sgl_alloc_index = 0;
3037         return 0;
3038 }
3039
3040 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3041 {
3042         int i, new_cid;
3043
3044         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3045                                   GFP_KERNEL);
3046         if (!phba->cid_array) {
3047                 shost_printk(KERN_ERR, phba->shost,
3048                              "Failed to allocate memory in "
3049                              "hba_setup_cid_tbls\n");
3050                 return -ENOMEM;
3051         }
3052         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3053                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3054         if (!phba->ep_array) {
3055                 shost_printk(KERN_ERR, phba->shost,
3056                              "Failed to allocate memory in "
3057                              "hba_setup_cid_tbls \n");
3058                 kfree(phba->cid_array);
3059                 return -ENOMEM;
3060         }
3061         new_cid = phba->fw_config.iscsi_cid_start;
3062         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3063                 phba->cid_array[i] = new_cid;
3064                 new_cid += 2;
3065         }
3066         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3067         return 0;
3068 }
3069
3070 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3071 {
3072         struct be_ctrl_info *ctrl = &phba->ctrl;
3073         struct hwi_controller *phwi_ctrlr;
3074         struct hwi_context_memory *phwi_context;
3075         struct be_queue_info *eq;
3076         u8 __iomem *addr;
3077         u32 reg, i;
3078         u32 enabled;
3079
3080         phwi_ctrlr = phba->phwi_ctrlr;
3081         phwi_context = phwi_ctrlr->phwi_ctxt;
3082
3083         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3084                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3085         reg = ioread32(addr);
3086         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3087
3088         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3089         if (!enabled) {
3090                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3091                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3092                 iowrite32(reg, addr);
3093                 for (i = 0; i <= phba->num_cpus; i++) {
3094                         eq = &phwi_context->be_eq[i].q;
3095                         SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3096                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3097                 }
3098         } else
3099                 shost_printk(KERN_WARNING, phba->shost,
3100                              "In hwi_enable_intr, Not Enabled \n");
3101         return true;
3102 }
3103
3104 static void hwi_disable_intr(struct beiscsi_hba *phba)
3105 {
3106         struct be_ctrl_info *ctrl = &phba->ctrl;
3107
3108         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3109         u32 reg = ioread32(addr);
3110
3111         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3112         if (enabled) {
3113                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3114                 iowrite32(reg, addr);
3115         } else
3116                 shost_printk(KERN_WARNING, phba->shost,
3117                              "In hwi_disable_intr, Already Disabled \n");
3118 }
3119
3120 static int beiscsi_init_port(struct beiscsi_hba *phba)
3121 {
3122         int ret;
3123
3124         ret = beiscsi_init_controller(phba);
3125         if (ret < 0) {
3126                 shost_printk(KERN_ERR, phba->shost,
3127                              "beiscsi_dev_probe - Failed in"
3128                              "beiscsi_init_controller \n");
3129                 return ret;
3130         }
3131         ret = beiscsi_init_sgl_handle(phba);
3132         if (ret < 0) {
3133                 shost_printk(KERN_ERR, phba->shost,
3134                              "beiscsi_dev_probe - Failed in"
3135                              "beiscsi_init_sgl_handle \n");
3136                 goto do_cleanup_ctrlr;
3137         }
3138
3139         if (hba_setup_cid_tbls(phba)) {
3140                 shost_printk(KERN_ERR, phba->shost,
3141                              "Failed in hba_setup_cid_tbls\n");
3142                 if (ring_mode)
3143                         kfree(phba->sgl_hndl_array);
3144                 kfree(phba->io_sgl_hndl_base);
3145                 kfree(phba->eh_sgl_hndl_base);
3146                 goto do_cleanup_ctrlr;
3147         }
3148
3149         return ret;
3150
3151 do_cleanup_ctrlr:
3152         hwi_cleanup(phba);
3153         return ret;
3154 }
3155
3156 static void hwi_purge_eq(struct beiscsi_hba *phba)
3157 {
3158         struct hwi_controller *phwi_ctrlr;
3159         struct hwi_context_memory *phwi_context;
3160         struct be_queue_info *eq;
3161         struct be_eq_entry *eqe = NULL;
3162         int i, eq_msix;
3163
3164         phwi_ctrlr = phba->phwi_ctrlr;
3165         phwi_context = phwi_ctrlr->phwi_ctxt;
3166         if (phba->msix_enabled)
3167                 eq_msix = 1;
3168         else
3169                 eq_msix = 0;
3170
3171         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3172                 eq = &phwi_context->be_eq[i].q;
3173                 eqe = queue_tail_node(eq);
3174
3175                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3176                                         & EQE_VALID_MASK) {
3177                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3178                         queue_tail_inc(eq);
3179                         eqe = queue_tail_node(eq);
3180                 }
3181         }
3182 }
3183
3184 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3185 {
3186         unsigned char mgmt_status;
3187
3188         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3189         if (mgmt_status)
3190                 shost_printk(KERN_WARNING, phba->shost,
3191                              "mgmt_epfw_cleanup FAILED \n");
3192         hwi_cleanup(phba);
3193         hwi_purge_eq(phba);
3194         if (ring_mode)
3195                 kfree(phba->sgl_hndl_array);
3196         kfree(phba->io_sgl_hndl_base);
3197         kfree(phba->eh_sgl_hndl_base);
3198         kfree(phba->cid_array);
3199         kfree(phba->ep_array);
3200 }
3201
3202 void
3203 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3204                            struct beiscsi_offload_params *params)
3205 {
3206         struct wrb_handle *pwrb_handle;
3207         struct iscsi_target_context_update_wrb *pwrb = NULL;
3208         struct be_mem_descriptor *mem_descr;
3209         struct beiscsi_hba *phba = beiscsi_conn->phba;
3210         u32 doorbell = 0;
3211
3212         /*
3213          * We can always use 0 here because it is reserved by libiscsi for
3214          * login/startup related tasks.
3215          */
3216         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3217                                        phba->fw_config.iscsi_cid_start));
3218         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3219         memset(pwrb, 0, sizeof(*pwrb));
3220         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3221                       max_burst_length, pwrb, params->dw[offsetof
3222                       (struct amap_beiscsi_offload_params,
3223                       max_burst_length) / 32]);
3224         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3225                       max_send_data_segment_length, pwrb,
3226                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3227                       max_send_data_segment_length) / 32]);
3228         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3229                       first_burst_length,
3230                       pwrb,
3231                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3232                       first_burst_length) / 32]);
3233
3234         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3235                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3236                       erl) / 32] & OFFLD_PARAMS_ERL));
3237         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3238                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3239                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3240         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3241                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3242                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3243         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3244                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3245                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3246         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3247                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3248                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3249         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3250                       pwrb,
3251                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3252                       exp_statsn) / 32] + 1));
3253         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3254                       0x7);
3255         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3256                       pwrb, pwrb_handle->wrb_index);
3257         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3258                       pwrb, pwrb_handle->nxt_wrb_index);
3259         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3260                         session_state, pwrb, 0);
3261         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3262                       pwrb, 1);
3263         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3264                       pwrb, 0);
3265         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3266                       0);
3267
3268         mem_descr = phba->init_mem;
3269         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3270
3271         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3272                         pad_buffer_addr_hi, pwrb,
3273                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3274         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3275                         pad_buffer_addr_lo, pwrb,
3276                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3277
3278         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3279
3280         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3281         if (!ring_mode)
3282                 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3283                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3284         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3285
3286         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3287 }
3288
3289 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3290                               int *index, int *age)
3291 {
3292         *index = (int)itt;
3293         if (age)
3294                 *age = conn->session->age;
3295 }
3296
3297 /**
3298  * beiscsi_alloc_pdu - allocates pdu and related resources
3299  * @task: libiscsi task
3300  * @opcode: opcode of pdu for task
3301  *
3302  * This is called with the session lock held. It will allocate
3303  * the wrb and sgl if needed for the command. And it will prep
3304  * the pdu's itt. beiscsi_parse_pdu will later translate
3305  * the pdu itt to the libiscsi task itt.
3306  */
3307 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3308 {
3309         struct beiscsi_io_task *io_task = task->dd_data;
3310         struct iscsi_conn *conn = task->conn;
3311         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3312         struct beiscsi_hba *phba = beiscsi_conn->phba;
3313         struct hwi_wrb_context *pwrb_context;
3314         struct hwi_controller *phwi_ctrlr;
3315         itt_t itt;
3316         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3317         dma_addr_t paddr;
3318
3319         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3320                                           GFP_KERNEL, &paddr);
3321         if (!io_task->cmd_bhs)
3322                 return -ENOMEM;
3323         io_task->bhs_pa.u.a64.address = paddr;
3324         io_task->libiscsi_itt = (itt_t)task->itt;
3325         io_task->pwrb_handle = alloc_wrb_handle(phba,
3326                                                 beiscsi_conn->beiscsi_conn_cid -
3327                                                 phba->fw_config.iscsi_cid_start
3328                                                 );
3329         io_task->conn = beiscsi_conn;
3330
3331         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3332         task->hdr_max = sizeof(struct be_cmd_bhs);
3333
3334         if (task->sc) {
3335                 spin_lock(&phba->io_sgl_lock);
3336                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3337                 spin_unlock(&phba->io_sgl_lock);
3338                 if (!io_task->psgl_handle)
3339                         goto free_hndls;
3340         } else {
3341                 io_task->scsi_cmnd = NULL;
3342                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3343                         if (!beiscsi_conn->login_in_progress) {
3344                                 spin_lock(&phba->mgmt_sgl_lock);
3345                                 io_task->psgl_handle = (struct sgl_handle *)
3346                                                 alloc_mgmt_sgl_handle(phba);
3347                                 spin_unlock(&phba->mgmt_sgl_lock);
3348                                 if (!io_task->psgl_handle)
3349                                         goto free_hndls;
3350
3351                                 beiscsi_conn->login_in_progress = 1;
3352                                 beiscsi_conn->plogin_sgl_handle =
3353                                                         io_task->psgl_handle;
3354                         } else {
3355                                 io_task->psgl_handle =
3356                                                 beiscsi_conn->plogin_sgl_handle;
3357                         }
3358                 } else {
3359                         spin_lock(&phba->mgmt_sgl_lock);
3360                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3361                         spin_unlock(&phba->mgmt_sgl_lock);
3362                         if (!io_task->psgl_handle)
3363                                 goto free_hndls;
3364                 }
3365         }
3366         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3367                                  wrb_index << 16) | (unsigned int)
3368                                 (io_task->psgl_handle->sgl_index));
3369         if (ring_mode) {
3370                 phba->sgl_hndl_array[io_task->psgl_handle->sgl_index -
3371                                      phba->fw_config.iscsi_icd_start] =
3372                                      io_task->psgl_handle;
3373                 io_task->psgl_handle->task = task;
3374                 io_task->psgl_handle->cid = beiscsi_conn->beiscsi_conn_cid  -
3375                                             phba->fw_config.iscsi_cid_start;
3376         } else
3377                 io_task->pwrb_handle->pio_handle = task;
3378
3379         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3380         return 0;
3381
3382 free_hndls:
3383         phwi_ctrlr = phba->phwi_ctrlr;
3384         pwrb_context = &phwi_ctrlr->wrb_context[
3385                         beiscsi_conn->beiscsi_conn_cid -
3386                         phba->fw_config.iscsi_cid_start];
3387         free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3388         io_task->pwrb_handle = NULL;
3389         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3390                       io_task->bhs_pa.u.a64.address);
3391         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3392         return -ENOMEM;
3393 }
3394
3395 static void beiscsi_cleanup_task(struct iscsi_task *task)
3396 {
3397         struct beiscsi_io_task *io_task = task->dd_data;
3398         struct iscsi_conn *conn = task->conn;
3399         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3400         struct beiscsi_hba *phba = beiscsi_conn->phba;
3401         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3402         struct hwi_wrb_context *pwrb_context;
3403         struct hwi_controller *phwi_ctrlr;
3404
3405         phwi_ctrlr = phba->phwi_ctrlr;
3406         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3407                         - phba->fw_config.iscsi_cid_start];
3408         if (io_task->pwrb_handle) {
3409                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3410                 io_task->pwrb_handle = NULL;
3411         }
3412
3413         if (io_task->cmd_bhs) {
3414                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3415                               io_task->bhs_pa.u.a64.address);
3416         }
3417
3418         if (task->sc) {
3419                 if (io_task->psgl_handle) {
3420                         spin_lock(&phba->io_sgl_lock);
3421                         free_io_sgl_handle(phba, io_task->psgl_handle);
3422                         spin_unlock(&phba->io_sgl_lock);
3423                         io_task->psgl_handle = NULL;
3424                 }
3425         } else {
3426                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3427                         return;
3428                 if (io_task->psgl_handle) {
3429                         spin_lock(&phba->mgmt_sgl_lock);
3430                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3431                         spin_unlock(&phba->mgmt_sgl_lock);
3432                         io_task->psgl_handle = NULL;
3433                 }
3434         }
3435 }
3436
3437 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3438                           unsigned int num_sg, unsigned int xferlen,
3439                           unsigned int writedir)
3440 {
3441
3442         struct beiscsi_io_task *io_task = task->dd_data;
3443         struct iscsi_conn *conn = task->conn;
3444         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3445         struct beiscsi_hba *phba = beiscsi_conn->phba;
3446         struct iscsi_wrb *pwrb = NULL;
3447         unsigned int doorbell = 0;
3448
3449         pwrb = io_task->pwrb_handle->pwrb;
3450         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3451         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3452
3453         if (writedir) {
3454                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3455                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3456                               &io_task->cmd_bhs->iscsi_data_pdu,
3457                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3458                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3459                               &io_task->cmd_bhs->iscsi_data_pdu,
3460                               ISCSI_OPCODE_SCSI_DATA_OUT);
3461                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3462                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3463                 if (ring_mode)
3464                         io_task->psgl_handle->type = INI_WR_CMD;
3465                 else
3466                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3467                                       INI_WR_CMD);
3468                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3469         } else {
3470                 if (ring_mode)
3471                         io_task->psgl_handle->type = INI_RD_CMD;
3472                 else
3473                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3474                                       INI_RD_CMD);
3475                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3476         }
3477         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3478                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3479                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3480
3481         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3482                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3483                                   lun[0]));
3484         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3485         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3486                       io_task->pwrb_handle->wrb_index);
3487         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3488                       be32_to_cpu(task->cmdsn));
3489         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3490                       io_task->psgl_handle->sgl_index);
3491
3492         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3493
3494         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3495                       io_task->pwrb_handle->nxt_wrb_index);
3496         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3497
3498         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3499         if (!ring_mode)
3500                 doorbell |= (io_task->pwrb_handle->wrb_index &
3501                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3502         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3503
3504         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3505         return 0;
3506 }
3507
3508 static int beiscsi_mtask(struct iscsi_task *task)
3509 {
3510         struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3511         struct iscsi_conn *conn = task->conn;
3512         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3513         struct beiscsi_hba *phba = beiscsi_conn->phba;
3514         struct iscsi_session *session;
3515         struct iscsi_wrb *pwrb = NULL;
3516         struct hwi_controller *phwi_ctrlr;
3517         struct hwi_wrb_context *pwrb_context;
3518         struct wrb_handle *pwrb_handle;
3519         unsigned int doorbell = 0;
3520         unsigned int i, cid;
3521         struct iscsi_task *aborted_task;
3522
3523         cid = beiscsi_conn->beiscsi_conn_cid;
3524         pwrb = io_task->pwrb_handle->pwrb;
3525         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3526                       be32_to_cpu(task->cmdsn));
3527         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3528                       io_task->pwrb_handle->wrb_index);
3529         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3530                       io_task->psgl_handle->sgl_index);
3531
3532         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3533         case ISCSI_OP_LOGIN:
3534                 if (ring_mode)
3535                         io_task->psgl_handle->type = TGT_DM_CMD;
3536                 else
3537                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3538                                       TGT_DM_CMD);
3539                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3540                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3541                 hwi_write_buffer(pwrb, task);
3542                 break;
3543         case ISCSI_OP_NOOP_OUT:
3544                 if (ring_mode)
3545                         io_task->psgl_handle->type = INI_RD_CMD;
3546                 else
3547                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3548                                       INI_RD_CMD);
3549                 if (task->hdr->ttt == ISCSI_RESERVED_TAG)
3550                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3551                 else
3552                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
3553
3554                 hwi_write_buffer(pwrb, task);
3555                 break;
3556         case ISCSI_OP_TEXT:
3557                 if (ring_mode)
3558                         io_task->psgl_handle->type = INI_WR_CMD;
3559                 else
3560                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3561                                       INI_WR_CMD);
3562                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3563                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3564                 hwi_write_buffer(pwrb, task);
3565                 break;
3566         case ISCSI_OP_SCSI_TMFUNC:
3567                 session = conn->session;
3568                 i = ((struct iscsi_tm *)task->hdr)->rtt;
3569                 phwi_ctrlr = phba->phwi_ctrlr;
3570                 pwrb_context = &phwi_ctrlr->wrb_context[cid -
3571                                             phba->fw_config.iscsi_cid_start];
3572                 pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i)
3573                                                                 >> 16];
3574                 aborted_task = pwrb_handle->pio_handle;
3575                  if (!aborted_task)
3576                         return 0;
3577
3578                 aborted_io_task = aborted_task->dd_data;
3579                 if (!aborted_io_task->scsi_cmnd)
3580                         return 0;
3581
3582                 mgmt_invalidate_icds(phba,
3583                                      aborted_io_task->psgl_handle->sgl_index,
3584                                      cid);
3585                 if (ring_mode)
3586                         io_task->psgl_handle->type = INI_TMF_CMD;
3587                 else
3588                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3589                                       INI_TMF_CMD);
3590                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3591                 hwi_write_buffer(pwrb, task);
3592                 break;
3593         case ISCSI_OP_LOGOUT:
3594                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3595                 if (ring_mode)
3596                         io_task->psgl_handle->type = HWH_TYPE_LOGOUT;
3597                 else
3598                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3599                                 HWH_TYPE_LOGOUT);
3600                 hwi_write_buffer(pwrb, task);
3601                 break;
3602
3603         default:
3604                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3605                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3606                 return -EINVAL;
3607         }
3608
3609         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3610                       task->data_count);
3611         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3612                       io_task->pwrb_handle->nxt_wrb_index);
3613         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3614
3615         doorbell |= cid & DB_WRB_POST_CID_MASK;
3616         if (!ring_mode)
3617                 doorbell |= (io_task->pwrb_handle->wrb_index &
3618                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3619         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3620         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3621         return 0;
3622 }
3623
3624 static int beiscsi_task_xmit(struct iscsi_task *task)
3625 {
3626         struct iscsi_conn *conn = task->conn;
3627         struct beiscsi_io_task *io_task = task->dd_data;
3628         struct scsi_cmnd *sc = task->sc;
3629         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3630         struct scatterlist *sg;
3631         int num_sg;
3632         unsigned int  writedir = 0, xferlen = 0;
3633
3634         SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3635                  "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3636                  task, conn, beiscsi_conn);
3637         if (!sc)
3638                 return beiscsi_mtask(task);
3639
3640         io_task->scsi_cmnd = sc;
3641         num_sg = scsi_dma_map(sc);
3642         if (num_sg < 0) {
3643                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3644                 return num_sg;
3645         }
3646         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3647                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3648         xferlen = scsi_bufflen(sc);
3649         sg = scsi_sglist(sc);
3650         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3651                 writedir = 1;
3652                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3653                          task->imm_count);
3654         } else
3655                 writedir = 0;
3656         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3657 }
3658
3659
3660 static void beiscsi_remove(struct pci_dev *pcidev)
3661 {
3662         struct beiscsi_hba *phba = NULL;
3663         struct hwi_controller *phwi_ctrlr;
3664         struct hwi_context_memory *phwi_context;
3665         struct be_eq_obj *pbe_eq;
3666         unsigned int i, msix_vec;
3667
3668         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3669         if (!phba) {
3670                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3671                 return;
3672         }
3673
3674         phwi_ctrlr = phba->phwi_ctrlr;
3675         phwi_context = phwi_ctrlr->phwi_ctxt;
3676         hwi_disable_intr(phba);
3677         if (phba->msix_enabled) {
3678                 for (i = 0; i <= phba->num_cpus; i++) {
3679                         msix_vec = phba->msix_entries[i].vector;
3680                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3681                 }
3682         } else
3683                 if (phba->pcidev->irq)
3684                         free_irq(phba->pcidev->irq, phba);
3685         pci_disable_msix(phba->pcidev);
3686         destroy_workqueue(phba->wq);
3687         if (blk_iopoll_enabled)
3688                 for (i = 0; i < phba->num_cpus; i++) {
3689                         pbe_eq = &phwi_context->be_eq[i];
3690                         blk_iopoll_disable(&pbe_eq->iopoll);
3691                 }
3692
3693         beiscsi_clean_port(phba);
3694         beiscsi_free_mem(phba);
3695         beiscsi_unmap_pci_function(phba);
3696         pci_free_consistent(phba->pcidev,
3697                             phba->ctrl.mbox_mem_alloced.size,
3698                             phba->ctrl.mbox_mem_alloced.va,
3699                             phba->ctrl.mbox_mem_alloced.dma);
3700         iscsi_host_remove(phba->shost);
3701         pci_dev_put(phba->pcidev);
3702         iscsi_host_free(phba->shost);
3703 }
3704
3705 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3706 {
3707         int i, status;
3708
3709         for (i = 0; i <= phba->num_cpus; i++)
3710                 phba->msix_entries[i].entry = i;
3711
3712         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3713                                  (phba->num_cpus + 1));
3714         if (!status)
3715                 phba->msix_enabled = true;
3716
3717         return;
3718 }
3719
3720 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3721                                 const struct pci_device_id *id)
3722 {
3723         struct beiscsi_hba *phba = NULL;
3724         struct hwi_controller *phwi_ctrlr;
3725         struct hwi_context_memory *phwi_context;
3726         struct be_eq_obj *pbe_eq;
3727         int ret, msix_vec, num_cpus, i;
3728
3729         ret = beiscsi_enable_pci(pcidev);
3730         if (ret < 0) {
3731                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3732                              "Failed to enable pci device \n");
3733                 return ret;
3734         }
3735
3736         phba = beiscsi_hba_alloc(pcidev);
3737         if (!phba) {
3738                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3739                         " Failed in beiscsi_hba_alloc \n");
3740                 goto disable_pci;
3741         }
3742         SE_DEBUG(DBG_LVL_8, " phba = %p \n", phba);
3743
3744         if (enable_msix)
3745                 num_cpus = find_num_cpus();
3746         else
3747                 num_cpus = 1;
3748         phba->num_cpus = num_cpus;
3749         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3750
3751         if (enable_msix)
3752                 beiscsi_msix_enable(phba);
3753         ret = be_ctrl_init(phba, pcidev);
3754         if (ret) {
3755                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3756                                 "Failed in be_ctrl_init\n");
3757                 goto hba_free;
3758         }
3759
3760         spin_lock_init(&phba->io_sgl_lock);
3761         spin_lock_init(&phba->mgmt_sgl_lock);
3762         spin_lock_init(&phba->isr_lock);
3763         ret = mgmt_get_fw_config(&phba->ctrl, phba);
3764         if (ret != 0) {
3765                 shost_printk(KERN_ERR, phba->shost,
3766                              "Error getting fw config\n");
3767                 goto free_port;
3768         }
3769         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
3770         phba->shost->can_queue = phba->params.ios_per_ctrl;
3771         beiscsi_get_params(phba);
3772         ret = beiscsi_init_port(phba);
3773         if (ret < 0) {
3774                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3775                              "Failed in beiscsi_init_port\n");
3776                 goto free_port;
3777         }
3778
3779         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3780                  phba->shost->host_no);
3781         phba->wq = create_workqueue(phba->wq_name);
3782         if (!phba->wq) {
3783                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3784                                 "Failed to allocate work queue\n");
3785                 goto free_twq;
3786         }
3787
3788         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3789
3790         phwi_ctrlr = phba->phwi_ctrlr;
3791         phwi_context = phwi_ctrlr->phwi_ctxt;
3792         if (blk_iopoll_enabled) {
3793                 for (i = 0; i < phba->num_cpus; i++) {
3794                         pbe_eq = &phwi_context->be_eq[i];
3795                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3796                                         be_iopoll);
3797                         blk_iopoll_enable(&pbe_eq->iopoll);
3798                 }
3799         }
3800         ret = beiscsi_init_irqs(phba);
3801         if (ret < 0) {
3802                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3803                              "Failed to beiscsi_init_irqs\n");
3804                 goto free_blkenbld;
3805         }
3806         ret = hwi_enable_intr(phba);
3807         if (ret < 0) {
3808                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3809                              "Failed to hwi_enable_intr\n");
3810                 goto free_ctrlr;
3811         }
3812         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3813         return 0;
3814
3815 free_ctrlr:
3816         if (phba->msix_enabled) {
3817                 for (i = 0; i <= phba->num_cpus; i++) {
3818                         msix_vec = phba->msix_entries[i].vector;
3819                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3820                 }
3821         } else
3822                 if (phba->pcidev->irq)
3823                         free_irq(phba->pcidev->irq, phba);
3824         pci_disable_msix(phba->pcidev);
3825 free_blkenbld:
3826         destroy_workqueue(phba->wq);
3827         if (blk_iopoll_enabled)
3828                 for (i = 0; i < phba->num_cpus; i++) {
3829                         pbe_eq = &phwi_context->be_eq[i];
3830                         blk_iopoll_disable(&pbe_eq->iopoll);
3831                 }
3832 free_twq:
3833         beiscsi_clean_port(phba);
3834         beiscsi_free_mem(phba);
3835 free_port:
3836         pci_free_consistent(phba->pcidev,
3837                             phba->ctrl.mbox_mem_alloced.size,
3838                             phba->ctrl.mbox_mem_alloced.va,
3839                            phba->ctrl.mbox_mem_alloced.dma);
3840         beiscsi_unmap_pci_function(phba);
3841 hba_free:
3842         iscsi_host_remove(phba->shost);
3843         pci_dev_put(phba->pcidev);
3844         iscsi_host_free(phba->shost);
3845 disable_pci:
3846         pci_disable_device(pcidev);
3847         return ret;
3848 }
3849
3850 struct iscsi_transport beiscsi_iscsi_transport = {
3851         .owner = THIS_MODULE,
3852         .name = DRV_NAME,
3853         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3854                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3855         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3856                 ISCSI_MAX_XMIT_DLENGTH |
3857                 ISCSI_HDRDGST_EN |
3858                 ISCSI_DATADGST_EN |
3859                 ISCSI_INITIAL_R2T_EN |
3860                 ISCSI_MAX_R2T |
3861                 ISCSI_IMM_DATA_EN |
3862                 ISCSI_FIRST_BURST |
3863                 ISCSI_MAX_BURST |
3864                 ISCSI_PDU_INORDER_EN |
3865                 ISCSI_DATASEQ_INORDER_EN |
3866                 ISCSI_ERL |
3867                 ISCSI_CONN_PORT |
3868                 ISCSI_CONN_ADDRESS |
3869                 ISCSI_EXP_STATSN |
3870                 ISCSI_PERSISTENT_PORT |
3871                 ISCSI_PERSISTENT_ADDRESS |
3872                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3873                 ISCSI_USERNAME | ISCSI_PASSWORD |
3874                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3875                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3876                 ISCSI_LU_RESET_TMO |
3877                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3878                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3879         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3880                                 ISCSI_HOST_INITIATOR_NAME,
3881         .create_session = beiscsi_session_create,
3882         .destroy_session = beiscsi_session_destroy,
3883         .create_conn = beiscsi_conn_create,
3884         .bind_conn = beiscsi_conn_bind,
3885         .destroy_conn = iscsi_conn_teardown,
3886         .set_param = beiscsi_set_param,
3887         .get_conn_param = beiscsi_conn_get_param,
3888         .get_session_param = iscsi_session_get_param,
3889         .get_host_param = beiscsi_get_host_param,
3890         .start_conn = beiscsi_conn_start,
3891         .stop_conn = beiscsi_conn_stop,
3892         .send_pdu = iscsi_conn_send_pdu,
3893         .xmit_task = beiscsi_task_xmit,
3894         .cleanup_task = beiscsi_cleanup_task,
3895         .alloc_pdu = beiscsi_alloc_pdu,
3896         .parse_pdu_itt = beiscsi_parse_pdu,
3897         .get_stats = beiscsi_conn_get_stats,
3898         .ep_connect = beiscsi_ep_connect,
3899         .ep_poll = beiscsi_ep_poll,
3900         .ep_disconnect = beiscsi_ep_disconnect,
3901         .session_recovery_timedout = iscsi_session_recovery_timedout,
3902 };
3903
3904 static struct pci_driver beiscsi_pci_driver = {
3905         .name = DRV_NAME,
3906         .probe = beiscsi_dev_probe,
3907         .remove = beiscsi_remove,
3908         .id_table = beiscsi_pci_id_table
3909 };
3910
3911
3912 static int __init beiscsi_module_init(void)
3913 {
3914         int ret;
3915
3916         beiscsi_scsi_transport =
3917                         iscsi_register_transport(&beiscsi_iscsi_transport);
3918         if (!beiscsi_scsi_transport) {
3919                 SE_DEBUG(DBG_LVL_1,
3920                          "beiscsi_module_init - Unable to  register beiscsi"
3921                          "transport.\n");
3922                 ret = -ENOMEM;
3923         }
3924         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3925                  &beiscsi_iscsi_transport);
3926
3927         ret = pci_register_driver(&beiscsi_pci_driver);
3928         if (ret) {
3929                 SE_DEBUG(DBG_LVL_1,
3930                          "beiscsi_module_init - Unable to  register"
3931                          "beiscsi pci driver.\n");
3932                 goto unregister_iscsi_transport;
3933         }
3934         ring_mode = 0;
3935         return 0;
3936
3937 unregister_iscsi_transport:
3938         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3939         return ret;
3940 }
3941
3942 static void __exit beiscsi_module_exit(void)
3943 {
3944         pci_unregister_driver(&beiscsi_pci_driver);
3945         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3946 }
3947
3948 module_init(beiscsi_module_init);
3949 module_exit(beiscsi_module_exit);