d520fe875859fc1cd36c0554943f65458ab2e40c
[linux-2.6.git] / drivers / scsi / be2iscsi / be_main.c
1 /**
2  * Copyright (C) 2005 - 2009 ServerEngines
3  * All rights reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License version 2
7  * as published by the Free Software Foundation.  The full GNU General
8  * Public License is included in this distribution in the file called COPYING.
9  *
10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
11  *
12  * Contact Information:
13  * linux-drivers@serverengines.com
14  *
15  *  ServerEngines
16  * 209 N. Fair Oaks Ave
17  * Sunnyvale, CA 94085
18  *
19  */
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/interrupt.h>
23 #include <linux/blkdev.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/kernel.h>
27 #include <linux/semaphore.h>
28
29 #include <scsi/libiscsi.h>
30 #include <scsi/scsi_transport_iscsi.h>
31 #include <scsi/scsi_transport.h>
32 #include <scsi/scsi_cmnd.h>
33 #include <scsi/scsi_device.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi.h>
36 #include "be_main.h"
37 #include "be_iscsi.h"
38 #include "be_mgmt.h"
39
40 static unsigned int be_iopoll_budget = 10;
41 static unsigned int be_max_phys_size = 64;
42 static unsigned int enable_msix;
43
44 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
45 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
46 MODULE_AUTHOR("ServerEngines Corporation");
47 MODULE_LICENSE("GPL");
48 module_param(be_iopoll_budget, int, 0);
49 module_param(enable_msix, int, 0);
50 module_param(be_max_phys_size, uint, S_IRUGO);
51 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
52                                    "contiguous memory that can be allocated."
53                                    "Range is 16 - 128");
54
55 static int beiscsi_slave_configure(struct scsi_device *sdev)
56 {
57         blk_queue_max_segment_size(sdev->request_queue, 65536);
58         return 0;
59 }
60
61 static struct scsi_host_template beiscsi_sht = {
62         .module = THIS_MODULE,
63         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
64         .proc_name = DRV_NAME,
65         .queuecommand = iscsi_queuecommand,
66         .eh_abort_handler = iscsi_eh_abort,
67         .change_queue_depth = iscsi_change_queue_depth,
68         .slave_configure = beiscsi_slave_configure,
69         .target_alloc = iscsi_target_alloc,
70         .eh_device_reset_handler = iscsi_eh_device_reset,
71         .eh_target_reset_handler = iscsi_eh_target_reset,
72         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
73         .can_queue = BE2_IO_DEPTH,
74         .this_id = -1,
75         .max_sectors = BEISCSI_MAX_SECTORS,
76         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
77         .use_clustering = ENABLE_CLUSTERING,
78 };
79 static struct scsi_transport_template *beiscsi_scsi_transport;
80
81 /*------------------- PCI Driver operations and data ----------------- */
82 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
83         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
84         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
85         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
86         { 0 }
87 };
88 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
89
90 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
91 {
92         struct beiscsi_hba *phba;
93         struct Scsi_Host *shost;
94
95         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
96         if (!shost) {
97                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
98                         "iscsi_host_alloc failed \n");
99                 return NULL;
100         }
101         shost->dma_boundary = pcidev->dma_mask;
102         shost->max_id = BE2_MAX_SESSIONS;
103         shost->max_channel = 0;
104         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
105         shost->max_lun = BEISCSI_NUM_MAX_LUN;
106         shost->transportt = beiscsi_scsi_transport;
107
108         phba = iscsi_host_priv(shost);
109         memset(phba, 0, sizeof(*phba));
110         phba->shost = shost;
111         phba->pcidev = pci_dev_get(pcidev);
112
113         if (iscsi_host_add(shost, &phba->pcidev->dev))
114                 goto free_devices;
115         return phba;
116
117 free_devices:
118         pci_dev_put(phba->pcidev);
119         iscsi_host_free(phba->shost);
120         return NULL;
121 }
122
123 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
124 {
125         if (phba->csr_va) {
126                 iounmap(phba->csr_va);
127                 phba->csr_va = NULL;
128         }
129         if (phba->db_va) {
130                 iounmap(phba->db_va);
131                 phba->db_va = NULL;
132         }
133         if (phba->pci_va) {
134                 iounmap(phba->pci_va);
135                 phba->pci_va = NULL;
136         }
137 }
138
139 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
140                                 struct pci_dev *pcidev)
141 {
142         u8 __iomem *addr;
143
144         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
145                                pci_resource_len(pcidev, 2));
146         if (addr == NULL)
147                 return -ENOMEM;
148         phba->ctrl.csr = addr;
149         phba->csr_va = addr;
150         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
151
152         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
153         if (addr == NULL)
154                 goto pci_map_err;
155         phba->ctrl.db = addr;
156         phba->db_va = addr;
157         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
158
159         addr = ioremap_nocache(pci_resource_start(pcidev, 1),
160                                pci_resource_len(pcidev, 1));
161         if (addr == NULL)
162                 goto pci_map_err;
163         phba->ctrl.pcicfg = addr;
164         phba->pci_va = addr;
165         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
166         return 0;
167
168 pci_map_err:
169         beiscsi_unmap_pci_function(phba);
170         return -ENOMEM;
171 }
172
173 static int beiscsi_enable_pci(struct pci_dev *pcidev)
174 {
175         int ret;
176
177         ret = pci_enable_device(pcidev);
178         if (ret) {
179                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
180                         "failed. Returning -ENODEV\n");
181                 return ret;
182         }
183
184         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
185                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
186                 if (ret) {
187                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
188                         pci_disable_device(pcidev);
189                         return ret;
190                 }
191         }
192         return 0;
193 }
194
195 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
196 {
197         struct be_ctrl_info *ctrl = &phba->ctrl;
198         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
199         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
200         int status = 0;
201
202         ctrl->pdev = pdev;
203         status = beiscsi_map_pci_bars(phba, pdev);
204         if (status)
205                 return status;
206
207         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
208         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
209                                                   mbox_mem_alloc->size,
210                                                   &mbox_mem_alloc->dma);
211         if (!mbox_mem_alloc->va) {
212                 beiscsi_unmap_pci_function(phba);
213                 status = -ENOMEM;
214                 return status;
215         }
216
217         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
218         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
219         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
220         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
221         spin_lock_init(&ctrl->mbox_lock);
222         return status;
223 }
224
225 static void beiscsi_get_params(struct beiscsi_hba *phba)
226 {
227         phba->params.ios_per_ctrl = BE2_IO_DEPTH;
228         phba->params.cxns_per_ctrl = BE2_MAX_SESSIONS;
229         phba->params.asyncpdus_per_ctrl = BE2_ASYNCPDUS;
230         phba->params.icds_per_ctrl = BE2_MAX_ICDS / 2;
231         phba->params.num_sge_per_io = BE2_SGE;
232         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
233         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
234         phba->params.eq_timer = 64;
235         phba->params.num_eq_entries =
236             (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
237                                                                 512) + 1) * 512;
238         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
239                                 ? 1024 : phba->params.num_eq_entries;
240         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
241                  phba->params.num_eq_entries);
242         phba->params.num_cq_entries =
243             (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
244                                                                 512) + 1) * 512;
245         SE_DEBUG(DBG_LVL_8,
246                 "phba->params.num_cq_entries=%d BE2_CMDS_PER_CXN=%d"
247                 "BE2_LOGOUTS=%d BE2_TMFS=%d BE2_ASYNCPDUS=%d \n",
248                 phba->params.num_cq_entries, BE2_CMDS_PER_CXN,
249                 BE2_LOGOUTS, BE2_TMFS, BE2_ASYNCPDUS);
250         phba->params.wrbs_per_cxn = 256;
251 }
252
253 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
254                            unsigned int id, unsigned int clr_interrupt,
255                            unsigned int num_processed,
256                            unsigned char rearm, unsigned char event)
257 {
258         u32 val = 0;
259         val |= id & DB_EQ_RING_ID_MASK;
260         if (rearm)
261                 val |= 1 << DB_EQ_REARM_SHIFT;
262         if (clr_interrupt)
263                 val |= 1 << DB_EQ_CLR_SHIFT;
264         if (event)
265                 val |= 1 << DB_EQ_EVNT_SHIFT;
266         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
267         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
268 }
269
270 /**
271  * be_isr - The isr routine of the driver.
272  * @irq: Not used
273  * @dev_id: Pointer to host adapter structure
274  */
275 static irqreturn_t be_isr(int irq, void *dev_id)
276 {
277         struct beiscsi_hba *phba;
278         struct hwi_controller *phwi_ctrlr;
279         struct hwi_context_memory *phwi_context;
280         struct be_eq_entry *eqe = NULL;
281         struct be_queue_info *eq;
282         struct be_queue_info *cq;
283         unsigned long flags, index;
284         unsigned int num_eq_processed;
285         struct be_ctrl_info *ctrl;
286         int isr;
287
288         phba = dev_id;
289         if (!enable_msix) {
290                 ctrl = &phba->ctrl;;
291                 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
292                                (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
293                 if (!isr)
294                         return IRQ_NONE;
295         }
296
297         phwi_ctrlr = phba->phwi_ctrlr;
298         phwi_context = phwi_ctrlr->phwi_ctxt;
299         eq = &phwi_context->be_eq.q;
300         cq = &phwi_context->be_cq;
301         index = 0;
302         eqe = queue_tail_node(eq);
303         if (!eqe)
304                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
305
306         num_eq_processed = 0;
307         if (blk_iopoll_enabled) {
308                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
309                                         & EQE_VALID_MASK) {
310                         if (!blk_iopoll_sched_prep(&phba->iopoll))
311                                 blk_iopoll_sched(&phba->iopoll);
312
313                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
314                         queue_tail_inc(eq);
315                         eqe = queue_tail_node(eq);
316                         num_eq_processed++;
317                         SE_DEBUG(DBG_LVL_8, "Valid EQE\n");
318                 }
319                 if (num_eq_processed) {
320                         hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 0, 1);
321                         return IRQ_HANDLED;
322                 } else
323                         return IRQ_NONE;
324         } else {
325                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
326                                                 & EQE_VALID_MASK) {
327
328                         if (((eqe->dw[offsetof(struct amap_eq_entry,
329                              resource_id) / 32] &
330                              EQE_RESID_MASK) >> 16) != cq->id) {
331                                 spin_lock_irqsave(&phba->isr_lock, flags);
332                                 phba->todo_mcc_cq = 1;
333                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
334                         } else {
335                                 spin_lock_irqsave(&phba->isr_lock, flags);
336                                 phba->todo_cq = 1;
337                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
338                         }
339                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
340                         queue_tail_inc(eq);
341                         eqe = queue_tail_node(eq);
342                         num_eq_processed++;
343                 }
344                 if (phba->todo_cq || phba->todo_mcc_cq)
345                         queue_work(phba->wq, &phba->work_cqs);
346
347                 if (num_eq_processed) {
348                         hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 1, 1);
349                         return IRQ_HANDLED;
350                 } else
351                         return IRQ_NONE;
352         }
353 }
354
355 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
356 {
357         struct pci_dev *pcidev = phba->pcidev;
358         int ret;
359
360         ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, "beiscsi", phba);
361         if (ret) {
362                 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
363                              "Failed to register irq\\n");
364                 return ret;
365         }
366         return 0;
367 }
368
369 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
370                            unsigned int id, unsigned int num_processed,
371                            unsigned char rearm, unsigned char event)
372 {
373         u32 val = 0;
374         val |= id & DB_CQ_RING_ID_MASK;
375         if (rearm)
376                 val |= 1 << DB_CQ_REARM_SHIFT;
377         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
378         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
379 }
380
381 /*
382  * async pdus include
383  * a. unsolicited NOP-In (target initiated NOP-In)
384  * b. Async Messages
385  * c. Reject PDU
386  * d. Login response
387  * These headers arrive unprocessed by the EP firmware and iSCSI layer
388  * process them
389  */
390 static unsigned int
391 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
392                           struct beiscsi_hba *phba,
393                           unsigned short cid,
394                           struct pdu_base *ppdu,
395                           unsigned long pdu_len,
396                           void *pbuffer, unsigned long buf_len)
397 {
398         struct iscsi_conn *conn = beiscsi_conn->conn;
399         struct iscsi_session *session = conn->session;
400
401         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
402                                                 PDUBASE_OPCODE_MASK) {
403         case ISCSI_OP_NOOP_IN:
404                 pbuffer = NULL;
405                 buf_len = 0;
406                 break;
407         case ISCSI_OP_ASYNC_EVENT:
408                 break;
409         case ISCSI_OP_REJECT:
410                 WARN_ON(!pbuffer);
411                 WARN_ON(!(buf_len == 48));
412                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
413                 break;
414         case ISCSI_OP_LOGIN_RSP:
415                 break;
416         default:
417                 shost_printk(KERN_WARNING, phba->shost,
418                              "Unrecognized opcode 0x%x in async msg \n",
419                              (ppdu->
420                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
421                                                 & PDUBASE_OPCODE_MASK));
422                 return 1;
423         }
424
425         spin_lock_bh(&session->lock);
426         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
427         spin_unlock_bh(&session->lock);
428         return 0;
429 }
430
431 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
432 {
433         struct sgl_handle *psgl_handle;
434
435         if (phba->io_sgl_hndl_avbl) {
436                 SE_DEBUG(DBG_LVL_8,
437                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
438                          phba->io_sgl_alloc_index);
439                 psgl_handle = phba->io_sgl_hndl_base[phba->
440                                                 io_sgl_alloc_index];
441                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
442                 phba->io_sgl_hndl_avbl--;
443                 if (phba->io_sgl_alloc_index == (phba->params.ios_per_ctrl - 1))
444                         phba->io_sgl_alloc_index = 0;
445                 else
446                         phba->io_sgl_alloc_index++;
447         } else
448                 psgl_handle = NULL;
449         return psgl_handle;
450 }
451
452 static void
453 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
454 {
455         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
456                  phba->io_sgl_free_index);
457         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
458                 /*
459                  * this can happen if clean_task is called on a task that
460                  * failed in xmit_task or alloc_pdu.
461                  */
462                  SE_DEBUG(DBG_LVL_8,
463                          "Double Free in IO SGL io_sgl_free_index=%d,"
464                          "value there=%p \n", phba->io_sgl_free_index,
465                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
466                 return;
467         }
468         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
469         phba->io_sgl_hndl_avbl++;
470         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
471                 phba->io_sgl_free_index = 0;
472         else
473                 phba->io_sgl_free_index++;
474 }
475
476 /**
477  * alloc_wrb_handle - To allocate a wrb handle
478  * @phba: The hba pointer
479  * @cid: The cid to use for allocation
480  * @index: index allocation and wrb index
481  *
482  * This happens under session_lock until submission to chip
483  */
484 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
485                                     int index)
486 {
487         struct hwi_wrb_context *pwrb_context;
488         struct hwi_controller *phwi_ctrlr;
489         struct wrb_handle *pwrb_handle;
490
491         phwi_ctrlr = phba->phwi_ctrlr;
492         pwrb_context = &phwi_ctrlr->wrb_context[cid];
493         pwrb_handle = pwrb_context->pwrb_handle_base[index];
494         pwrb_handle->wrb_index = index;
495         pwrb_handle->nxt_wrb_index = index;
496         return pwrb_handle;
497 }
498
499 /**
500  * free_wrb_handle - To free the wrb handle back to pool
501  * @phba: The hba pointer
502  * @pwrb_context: The context to free from
503  * @pwrb_handle: The wrb_handle to free
504  *
505  * This happens under session_lock until submission to chip
506  */
507 static void
508 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
509                 struct wrb_handle *pwrb_handle)
510 {
511         SE_DEBUG(DBG_LVL_8,
512                  "FREE WRB: pwrb_handle=%p free_index=%d=0x%x"
513                  "wrb_handles_available=%d \n",
514                  pwrb_handle, pwrb_context->free_index,
515                  pwrb_context->free_index, pwrb_context->wrb_handles_available);
516 }
517
518 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
519 {
520         struct sgl_handle *psgl_handle;
521
522         if (phba->eh_sgl_hndl_avbl) {
523                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
524                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
525                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
526                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
527                 phba->eh_sgl_hndl_avbl--;
528                 if (phba->eh_sgl_alloc_index ==
529                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
530                      1))
531                         phba->eh_sgl_alloc_index = 0;
532                 else
533                         phba->eh_sgl_alloc_index++;
534         } else
535                 psgl_handle = NULL;
536         return psgl_handle;
537 }
538
539 void
540 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
541 {
542
543         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
544                 /*
545                  * this can happen if clean_task is called on a task that
546                  * failed in xmit_task or alloc_pdu.
547                  */
548                 SE_DEBUG(DBG_LVL_8,
549                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
550                          phba->eh_sgl_free_index);
551                 return;
552         }
553         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
554         phba->eh_sgl_hndl_avbl++;
555         if (phba->eh_sgl_free_index ==
556             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
557                 phba->eh_sgl_free_index = 0;
558         else
559                 phba->eh_sgl_free_index++;
560 }
561
562 static void
563 be_complete_io(struct beiscsi_conn *beiscsi_conn,
564                struct iscsi_task *task, struct sol_cqe *psol)
565 {
566         struct beiscsi_io_task *io_task = task->dd_data;
567         struct be_status_bhs *sts_bhs =
568                                 (struct be_status_bhs *)io_task->cmd_bhs;
569         struct iscsi_conn *conn = beiscsi_conn->conn;
570         unsigned int sense_len;
571         unsigned char *sense;
572         u32 resid = 0, exp_cmdsn, max_cmdsn;
573         u8 rsp, status, flags;
574
575         exp_cmdsn = be32_to_cpu(psol->
576                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
577                         & SOL_EXP_CMD_SN_MASK);
578         max_cmdsn = be32_to_cpu((psol->
579                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
580                         & SOL_EXP_CMD_SN_MASK) +
581                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
582                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
583         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
584                                                 & SOL_RESP_MASK) >> 16);
585         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
586                                                 & SOL_STS_MASK) >> 8);
587         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
588                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
589
590         task->sc->result = (DID_OK << 16) | status;
591         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
592                 task->sc->result = DID_ERROR << 16;
593                 goto unmap;
594         }
595
596         /* bidi not initially supported */
597         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
598                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
599                                 32] & SOL_RES_CNT_MASK);
600
601                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
602                         task->sc->result = DID_ERROR << 16;
603
604                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
605                         scsi_set_resid(task->sc, resid);
606                         if (!status && (scsi_bufflen(task->sc) - resid <
607                             task->sc->underflow))
608                                 task->sc->result = DID_ERROR << 16;
609                 }
610         }
611
612         if (status == SAM_STAT_CHECK_CONDITION) {
613                 sense = sts_bhs->sense_info + sizeof(unsigned short);
614                 sense_len =
615                     cpu_to_be16((unsigned short)(sts_bhs->sense_info[0]));
616                 memcpy(task->sc->sense_buffer, sense,
617                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
618         }
619         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
620                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
621                                                         & SOL_RES_CNT_MASK)
622                          conn->rxdata_octets += (psol->
623                               dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
624                                                         & SOL_RES_CNT_MASK);
625         }
626 unmap:
627         scsi_dma_unmap(io_task->scsi_cmnd);
628         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
629 }
630
631 static void
632 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
633                    struct iscsi_task *task, struct sol_cqe *psol)
634 {
635         struct iscsi_logout_rsp *hdr;
636         struct iscsi_conn *conn = beiscsi_conn->conn;
637
638         hdr = (struct iscsi_logout_rsp *)task->hdr;
639         hdr->t2wait = 5;
640         hdr->t2retain = 0;
641         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
642                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
643         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
644                                         32] & SOL_RESP_MASK);
645         hdr->exp_cmdsn = cpu_to_be32(psol->
646                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
647                                         & SOL_EXP_CMD_SN_MASK);
648         hdr->max_cmdsn = be32_to_cpu((psol->
649                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
650                                         & SOL_EXP_CMD_SN_MASK) +
651                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
652                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
653         hdr->hlength = 0;
654
655         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
656 }
657
658 static void
659 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
660                 struct iscsi_task *task, struct sol_cqe *psol)
661 {
662         struct iscsi_tm_rsp *hdr;
663         struct iscsi_conn *conn = beiscsi_conn->conn;
664
665         hdr = (struct iscsi_tm_rsp *)task->hdr;
666         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
667                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
668         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
669                                         32] & SOL_RESP_MASK);
670         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
671                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
672         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
673                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
674                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
675                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
676         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
677 }
678
679 static void
680 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
681                        struct beiscsi_hba *phba, struct sol_cqe *psol)
682 {
683         struct hwi_wrb_context *pwrb_context;
684         struct wrb_handle *pwrb_handle;
685         struct hwi_controller *phwi_ctrlr;
686         struct iscsi_conn *conn = beiscsi_conn->conn;
687         struct iscsi_session *session = conn->session;
688
689         phwi_ctrlr = phba->phwi_ctrlr;
690         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
691                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
692                                 SOL_CID_MASK) >> 6)];
693         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
694                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
695                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
696         spin_lock_bh(&session->lock);
697         free_wrb_handle(phba, pwrb_context, pwrb_handle);
698         spin_unlock_bh(&session->lock);
699 }
700
701 static void
702 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
703                        struct iscsi_task *task, struct sol_cqe *psol)
704 {
705         struct iscsi_nopin *hdr;
706         struct iscsi_conn *conn = beiscsi_conn->conn;
707
708         hdr = (struct iscsi_nopin *)task->hdr;
709         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
710                         & SOL_FLAGS_MASK) >> 24) | 0x80;
711         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
712                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
713         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
714                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
715                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
716                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
717         hdr->opcode = ISCSI_OP_NOOP_IN;
718         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
719 }
720
721 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
722                              struct beiscsi_hba *phba, struct sol_cqe *psol)
723 {
724         struct hwi_wrb_context *pwrb_context;
725         struct wrb_handle *pwrb_handle;
726         struct iscsi_wrb *pwrb = NULL;
727         struct hwi_controller *phwi_ctrlr;
728         struct iscsi_task *task;
729         struct beiscsi_io_task *io_task;
730         struct iscsi_conn *conn = beiscsi_conn->conn;
731         struct iscsi_session *session = conn->session;
732
733         phwi_ctrlr = phba->phwi_ctrlr;
734
735         pwrb_context = &phwi_ctrlr->
736                 wrb_context[((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
737                 & SOL_CID_MASK) >> 6)];
738         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
739                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
740                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
741
742         task = pwrb_handle->pio_handle;
743         io_task = task->dd_data;
744         spin_lock_bh(&session->lock);
745         pwrb = pwrb_handle->pwrb;
746         switch ((pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
747                  WRB_TYPE_MASK) >> 28) {
748         case HWH_TYPE_IO:
749         case HWH_TYPE_IO_RD:
750                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
751                     ISCSI_OP_NOOP_OUT) {
752                         be_complete_nopin_resp(beiscsi_conn, task, psol);
753                 } else
754                         be_complete_io(beiscsi_conn, task, psol);
755                 break;
756
757         case HWH_TYPE_LOGOUT:
758                 be_complete_logout(beiscsi_conn, task, psol);
759                 break;
760
761         case HWH_TYPE_LOGIN:
762                 SE_DEBUG(DBG_LVL_1,
763                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
764                          "- Solicited path \n");
765                 break;
766
767         case HWH_TYPE_TMF:
768                 be_complete_tmf(beiscsi_conn, task, psol);
769                 break;
770
771         case HWH_TYPE_NOP:
772                 be_complete_nopin_resp(beiscsi_conn, task, psol);
773                 break;
774
775         default:
776                 shost_printk(KERN_WARNING, phba->shost,
777                             "wrb_index 0x%x CID 0x%x\n",
778                             ((psol->dw[offsetof(struct amap_iscsi_wrb, type) /
779                                         32] & SOL_WRB_INDEX_MASK) >> 16),
780                             ((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
781                                         & SOL_CID_MASK) >> 6));
782                 break;
783         }
784
785         spin_unlock_bh(&session->lock);
786 }
787
788 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
789                                           *pasync_ctx, unsigned int is_header,
790                                           unsigned int host_write_ptr)
791 {
792         if (is_header)
793                 return &pasync_ctx->async_entry[host_write_ptr].
794                     header_busy_list;
795         else
796                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
797 }
798
799 static struct async_pdu_handle *
800 hwi_get_async_handle(struct beiscsi_hba *phba,
801                      struct beiscsi_conn *beiscsi_conn,
802                      struct hwi_async_pdu_context *pasync_ctx,
803                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
804 {
805         struct be_bus_address phys_addr;
806         struct list_head *pbusy_list;
807         struct async_pdu_handle *pasync_handle = NULL;
808         int buffer_len = 0;
809         unsigned char buffer_index = -1;
810         unsigned char is_header = 0;
811
812         phys_addr.u.a32.address_lo =
813             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
814             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
815                                                 & PDUCQE_DPL_MASK) >> 16);
816         phys_addr.u.a32.address_hi =
817             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
818
819         phys_addr.u.a64.address =
820                         *((unsigned long long *)(&phys_addr.u.a64.address));
821
822         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
823                         & PDUCQE_CODE_MASK) {
824         case UNSOL_HDR_NOTIFY:
825                 is_header = 1;
826
827                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
828                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
829                         index) / 32] & PDUCQE_INDEX_MASK));
830
831                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
832                                 pasync_ctx->async_header.pa_base.u.a64.address);
833
834                 buffer_index = buffer_len /
835                                 pasync_ctx->async_header.buffer_size;
836
837                 break;
838         case UNSOL_DATA_NOTIFY:
839                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
840                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
841                                         index) / 32] & PDUCQE_INDEX_MASK));
842                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
843                                         pasync_ctx->async_data.pa_base.u.
844                                         a64.address);
845                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
846                 break;
847         default:
848                 pbusy_list = NULL;
849                 shost_printk(KERN_WARNING, phba->shost,
850                         "Unexpected code=%d \n",
851                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
852                                         code) / 32] & PDUCQE_CODE_MASK);
853                 return NULL;
854         }
855
856         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
857         WARN_ON(list_empty(pbusy_list));
858         list_for_each_entry(pasync_handle, pbusy_list, link) {
859                 WARN_ON(pasync_handle->consumed);
860                 if (pasync_handle->index == buffer_index)
861                         break;
862         }
863
864         WARN_ON(!pasync_handle);
865
866         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid;
867         pasync_handle->is_header = is_header;
868         pasync_handle->buffer_len = ((pdpdu_cqe->
869                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
870                         & PDUCQE_DPL_MASK) >> 16);
871
872         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
873                         index) / 32] & PDUCQE_INDEX_MASK);
874         return pasync_handle;
875 }
876
877 static unsigned int
878 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
879                            unsigned int is_header, unsigned int cq_index)
880 {
881         struct list_head *pbusy_list;
882         struct async_pdu_handle *pasync_handle;
883         unsigned int num_entries, writables = 0;
884         unsigned int *pep_read_ptr, *pwritables;
885
886
887         if (is_header) {
888                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
889                 pwritables = &pasync_ctx->async_header.writables;
890                 num_entries = pasync_ctx->async_header.num_entries;
891         } else {
892                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
893                 pwritables = &pasync_ctx->async_data.writables;
894                 num_entries = pasync_ctx->async_data.num_entries;
895         }
896
897         while ((*pep_read_ptr) != cq_index) {
898                 (*pep_read_ptr)++;
899                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
900
901                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
902                                                      *pep_read_ptr);
903                 if (writables == 0)
904                         WARN_ON(list_empty(pbusy_list));
905
906                 if (!list_empty(pbusy_list)) {
907                         pasync_handle = list_entry(pbusy_list->next,
908                                                    struct async_pdu_handle,
909                                                    link);
910                         WARN_ON(!pasync_handle);
911                         pasync_handle->consumed = 1;
912                 }
913
914                 writables++;
915         }
916
917         if (!writables) {
918                 SE_DEBUG(DBG_LVL_1,
919                          "Duplicate notification received - index 0x%x!!\n",
920                          cq_index);
921                 WARN_ON(1);
922         }
923
924         *pwritables = *pwritables + writables;
925         return 0;
926 }
927
928 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
929                                        unsigned int cri)
930 {
931         struct hwi_controller *phwi_ctrlr;
932         struct hwi_async_pdu_context *pasync_ctx;
933         struct async_pdu_handle *pasync_handle, *tmp_handle;
934         struct list_head *plist;
935         unsigned int i = 0;
936
937         phwi_ctrlr = phba->phwi_ctrlr;
938         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
939
940         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
941
942         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
943                 list_del(&pasync_handle->link);
944
945                 if (i == 0) {
946                         list_add_tail(&pasync_handle->link,
947                                       &pasync_ctx->async_header.free_list);
948                         pasync_ctx->async_header.free_entries++;
949                         i++;
950                 } else {
951                         list_add_tail(&pasync_handle->link,
952                                       &pasync_ctx->async_data.free_list);
953                         pasync_ctx->async_data.free_entries++;
954                         i++;
955                 }
956         }
957
958         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
959         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
960         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
961         return 0;
962 }
963
964 static struct phys_addr *
965 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
966                      unsigned int is_header, unsigned int host_write_ptr)
967 {
968         struct phys_addr *pasync_sge = NULL;
969
970         if (is_header)
971                 pasync_sge = pasync_ctx->async_header.ring_base;
972         else
973                 pasync_sge = pasync_ctx->async_data.ring_base;
974
975         return pasync_sge + host_write_ptr;
976 }
977
978 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
979                                    unsigned int is_header)
980 {
981         struct hwi_controller *phwi_ctrlr;
982         struct hwi_async_pdu_context *pasync_ctx;
983         struct async_pdu_handle *pasync_handle;
984         struct list_head *pfree_link, *pbusy_list;
985         struct phys_addr *pasync_sge;
986         unsigned int ring_id, num_entries;
987         unsigned int host_write_num;
988         unsigned int writables;
989         unsigned int i = 0;
990         u32 doorbell = 0;
991
992         phwi_ctrlr = phba->phwi_ctrlr;
993         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
994
995         if (is_header) {
996                 num_entries = pasync_ctx->async_header.num_entries;
997                 writables = min(pasync_ctx->async_header.writables,
998                                 pasync_ctx->async_header.free_entries);
999                 pfree_link = pasync_ctx->async_header.free_list.next;
1000                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1001                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1002         } else {
1003                 num_entries = pasync_ctx->async_data.num_entries;
1004                 writables = min(pasync_ctx->async_data.writables,
1005                                 pasync_ctx->async_data.free_entries);
1006                 pfree_link = pasync_ctx->async_data.free_list.next;
1007                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1008                 ring_id = phwi_ctrlr->default_pdu_data.id;
1009         }
1010
1011         writables = (writables / 8) * 8;
1012         if (writables) {
1013                 for (i = 0; i < writables; i++) {
1014                         pbusy_list =
1015                             hwi_get_async_busy_list(pasync_ctx, is_header,
1016                                                     host_write_num);
1017                         pasync_handle =
1018                             list_entry(pfree_link, struct async_pdu_handle,
1019                                                                 link);
1020                         WARN_ON(!pasync_handle);
1021                         pasync_handle->consumed = 0;
1022
1023                         pfree_link = pfree_link->next;
1024
1025                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1026                                                 is_header, host_write_num);
1027
1028                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1029                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1030
1031                         list_move(&pasync_handle->link, pbusy_list);
1032
1033                         host_write_num++;
1034                         host_write_num = host_write_num % num_entries;
1035                 }
1036
1037                 if (is_header) {
1038                         pasync_ctx->async_header.host_write_ptr =
1039                                                         host_write_num;
1040                         pasync_ctx->async_header.free_entries -= writables;
1041                         pasync_ctx->async_header.writables -= writables;
1042                         pasync_ctx->async_header.busy_entries += writables;
1043                 } else {
1044                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1045                         pasync_ctx->async_data.free_entries -= writables;
1046                         pasync_ctx->async_data.writables -= writables;
1047                         pasync_ctx->async_data.busy_entries += writables;
1048                 }
1049
1050                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1051                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1052                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1053                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1054                                         << DB_DEF_PDU_CQPROC_SHIFT;
1055
1056                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1057         }
1058 }
1059
1060 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1061                                          struct beiscsi_conn *beiscsi_conn,
1062                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1063 {
1064         struct hwi_controller *phwi_ctrlr;
1065         struct hwi_async_pdu_context *pasync_ctx;
1066         struct async_pdu_handle *pasync_handle = NULL;
1067         unsigned int cq_index = -1;
1068
1069         phwi_ctrlr = phba->phwi_ctrlr;
1070         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1071
1072         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1073                                              pdpdu_cqe, &cq_index);
1074         BUG_ON(pasync_handle->is_header != 0);
1075         if (pasync_handle->consumed == 0)
1076                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1077                                            cq_index);
1078
1079         hwi_free_async_msg(phba, pasync_handle->cri);
1080         hwi_post_async_buffers(phba, pasync_handle->is_header);
1081 }
1082
1083 static unsigned int
1084 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1085                   struct beiscsi_hba *phba,
1086                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1087 {
1088         struct list_head *plist;
1089         struct async_pdu_handle *pasync_handle;
1090         void *phdr = NULL;
1091         unsigned int hdr_len = 0, buf_len = 0;
1092         unsigned int status, index = 0, offset = 0;
1093         void *pfirst_buffer = NULL;
1094         unsigned int num_buf = 0;
1095
1096         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1097
1098         list_for_each_entry(pasync_handle, plist, link) {
1099                 if (index == 0) {
1100                         phdr = pasync_handle->pbuffer;
1101                         hdr_len = pasync_handle->buffer_len;
1102                 } else {
1103                         buf_len = pasync_handle->buffer_len;
1104                         if (!num_buf) {
1105                                 pfirst_buffer = pasync_handle->pbuffer;
1106                                 num_buf++;
1107                         }
1108                         memcpy(pfirst_buffer + offset,
1109                                pasync_handle->pbuffer, buf_len);
1110                         offset = buf_len;
1111                 }
1112                 index++;
1113         }
1114
1115         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1116                                            beiscsi_conn->beiscsi_conn_cid,
1117                                            phdr, hdr_len, pfirst_buffer,
1118                                            buf_len);
1119
1120         if (status == 0)
1121                 hwi_free_async_msg(phba, cri);
1122         return 0;
1123 }
1124
1125 static unsigned int
1126 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1127                      struct beiscsi_hba *phba,
1128                      struct async_pdu_handle *pasync_handle)
1129 {
1130         struct hwi_async_pdu_context *pasync_ctx;
1131         struct hwi_controller *phwi_ctrlr;
1132         unsigned int bytes_needed = 0, status = 0;
1133         unsigned short cri = pasync_handle->cri;
1134         struct pdu_base *ppdu;
1135
1136         phwi_ctrlr = phba->phwi_ctrlr;
1137         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1138
1139         list_del(&pasync_handle->link);
1140         if (pasync_handle->is_header) {
1141                 pasync_ctx->async_header.busy_entries--;
1142                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1143                         hwi_free_async_msg(phba, cri);
1144                         BUG();
1145                 }
1146
1147                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1148                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1149                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1150                                 (unsigned short)pasync_handle->buffer_len;
1151                 list_add_tail(&pasync_handle->link,
1152                               &pasync_ctx->async_entry[cri].wait_queue.list);
1153
1154                 ppdu = pasync_handle->pbuffer;
1155                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1156                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1157                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1158                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1159                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1160
1161                 if (status == 0) {
1162                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1163                             bytes_needed;
1164
1165                         if (bytes_needed == 0)
1166                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1167                                                            pasync_ctx, cri);
1168                 }
1169         } else {
1170                 pasync_ctx->async_data.busy_entries--;
1171                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1172                         list_add_tail(&pasync_handle->link,
1173                                       &pasync_ctx->async_entry[cri].wait_queue.
1174                                       list);
1175                         pasync_ctx->async_entry[cri].wait_queue.
1176                                 bytes_received +=
1177                                 (unsigned short)pasync_handle->buffer_len;
1178
1179                         if (pasync_ctx->async_entry[cri].wait_queue.
1180                             bytes_received >=
1181                             pasync_ctx->async_entry[cri].wait_queue.
1182                             bytes_needed)
1183                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1184                                                            pasync_ctx, cri);
1185                 }
1186         }
1187         return status;
1188 }
1189
1190 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1191                                          struct beiscsi_hba *phba,
1192                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1193 {
1194         struct hwi_controller *phwi_ctrlr;
1195         struct hwi_async_pdu_context *pasync_ctx;
1196         struct async_pdu_handle *pasync_handle = NULL;
1197         unsigned int cq_index = -1;
1198
1199         phwi_ctrlr = phba->phwi_ctrlr;
1200         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1201         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1202                                              pdpdu_cqe, &cq_index);
1203
1204         if (pasync_handle->consumed == 0)
1205                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1206                                            cq_index);
1207         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1208         hwi_post_async_buffers(phba, pasync_handle->is_header);
1209 }
1210
1211 static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1212 {
1213         struct hwi_controller *phwi_ctrlr;
1214         struct hwi_context_memory *phwi_context;
1215         struct be_queue_info *cq;
1216         struct sol_cqe *sol;
1217         struct dmsg_cqe *dmsg;
1218         unsigned int num_processed = 0;
1219         unsigned int tot_nump = 0;
1220         struct beiscsi_conn *beiscsi_conn;
1221
1222         phwi_ctrlr = phba->phwi_ctrlr;
1223         phwi_context = phwi_ctrlr->phwi_ctxt;
1224         cq = &phwi_context->be_cq;
1225         sol = queue_tail_node(cq);
1226
1227         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1228                CQE_VALID_MASK) {
1229                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1230
1231                 beiscsi_conn = phba->conn_table[(u32) (sol->
1232                                  dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1233                                  SOL_CID_MASK) >> 6];
1234
1235                 if (!beiscsi_conn || !beiscsi_conn->ep) {
1236                         shost_printk(KERN_WARNING, phba->shost,
1237                                      "Connection table empty for cid = %d\n",
1238                                      (u32)(sol->dw[offsetof(struct amap_sol_cqe,
1239                                      cid) / 32] & SOL_CID_MASK) >> 6);
1240                         return 0;
1241                 }
1242
1243                 if (num_processed >= 32) {
1244                         hwi_ring_cq_db(phba, phwi_context->be_cq.id,
1245                                         num_processed, 0, 0);
1246                         tot_nump += num_processed;
1247                         num_processed = 0;
1248                 }
1249
1250                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1251                         32] & CQE_CODE_MASK) {
1252                 case SOL_CMD_COMPLETE:
1253                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1254                         break;
1255                 case DRIVERMSG_NOTIFY:
1256                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1257                         dmsg = (struct dmsg_cqe *)sol;
1258                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1259                         break;
1260                 case UNSOL_HDR_NOTIFY:
1261                 case UNSOL_DATA_NOTIFY:
1262                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR/DATA_NOTIFY\n");
1263                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1264                                              (struct i_t_dpdu_cqe *)sol);
1265                         break;
1266                 case CXN_INVALIDATE_INDEX_NOTIFY:
1267                 case CMD_INVALIDATED_NOTIFY:
1268                 case CXN_INVALIDATE_NOTIFY:
1269                         SE_DEBUG(DBG_LVL_1,
1270                                  "Ignoring CQ Error notification for cmd/cxn"
1271                                  "invalidate\n");
1272                         break;
1273                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1274                 case CMD_KILLED_INVALID_STATSN_RCVD:
1275                 case CMD_KILLED_INVALID_R2T_RCVD:
1276                 case CMD_CXN_KILLED_LUN_INVALID:
1277                 case CMD_CXN_KILLED_ICD_INVALID:
1278                 case CMD_CXN_KILLED_ITT_INVALID:
1279                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1280                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1281                         SE_DEBUG(DBG_LVL_1,
1282                                  "CQ Error notification for cmd.. "
1283                                  "code %d cid 0x%x\n",
1284                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1285                                  32] & CQE_CODE_MASK,
1286                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1287                                  32] & SOL_CID_MASK));
1288                         break;
1289                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1290                         SE_DEBUG(DBG_LVL_1,
1291                                  "Digest error on def pdu ring, dropping..\n");
1292                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1293                                              (struct i_t_dpdu_cqe *) sol);
1294                         break;
1295                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1296                 case CXN_KILLED_BURST_LEN_MISMATCH:
1297                 case CXN_KILLED_AHS_RCVD:
1298                 case CXN_KILLED_HDR_DIGEST_ERR:
1299                 case CXN_KILLED_UNKNOWN_HDR:
1300                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1301                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1302                 case CXN_KILLED_TIMED_OUT:
1303                 case CXN_KILLED_FIN_RCVD:
1304                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1305                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1306                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1307                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1308                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1309                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, resetting CID "
1310                                  "0x%x...\n",
1311                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1312                                  32] & CQE_CODE_MASK,
1313                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1314                                  32] & CQE_CID_MASK);
1315                         iscsi_conn_failure(beiscsi_conn->conn,
1316                                            ISCSI_ERR_CONN_FAILED);
1317                         break;
1318                 case CXN_KILLED_RST_SENT:
1319                 case CXN_KILLED_RST_RCVD:
1320                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset received/sent "
1321                                  "on CID 0x%x...\n",
1322                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1323                                  32] & CQE_CODE_MASK,
1324                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1325                                  32] & CQE_CID_MASK);
1326                         iscsi_conn_failure(beiscsi_conn->conn,
1327                                            ISCSI_ERR_CONN_FAILED);
1328                         break;
1329                 default:
1330                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1331                                  "received on CID 0x%x...\n",
1332                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1333                                  32] & CQE_CODE_MASK,
1334                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1335                                  32] & CQE_CID_MASK);
1336                         break;
1337                 }
1338
1339                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1340                 queue_tail_inc(cq);
1341                 sol = queue_tail_node(cq);
1342                 num_processed++;
1343         }
1344
1345         if (num_processed > 0) {
1346                 tot_nump += num_processed;
1347                 hwi_ring_cq_db(phba, phwi_context->be_cq.id, num_processed,
1348                                1, 0);
1349         }
1350         return tot_nump;
1351 }
1352
1353 static void beiscsi_process_all_cqs(struct work_struct *work)
1354 {
1355         unsigned long flags;
1356         struct beiscsi_hba *phba =
1357             container_of(work, struct beiscsi_hba, work_cqs);
1358
1359         if (phba->todo_mcc_cq) {
1360                 spin_lock_irqsave(&phba->isr_lock, flags);
1361                 phba->todo_mcc_cq = 0;
1362                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1363                 SE_DEBUG(DBG_LVL_1, "MCC Interrupt Not expected \n");
1364         }
1365
1366         if (phba->todo_cq) {
1367                 spin_lock_irqsave(&phba->isr_lock, flags);
1368                 phba->todo_cq = 0;
1369                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1370                 beiscsi_process_cq(phba);
1371         }
1372 }
1373
1374 static int be_iopoll(struct blk_iopoll *iop, int budget)
1375 {
1376         static unsigned int ret;
1377         struct beiscsi_hba *phba;
1378
1379         phba = container_of(iop, struct beiscsi_hba, iopoll);
1380
1381         ret = beiscsi_process_cq(phba);
1382         if (ret < budget) {
1383                 struct hwi_controller *phwi_ctrlr;
1384                 struct hwi_context_memory *phwi_context;
1385
1386                 phwi_ctrlr = phba->phwi_ctrlr;
1387                 phwi_context = phwi_ctrlr->phwi_ctxt;
1388                 blk_iopoll_complete(iop);
1389                 hwi_ring_eq_db(phba, phwi_context->be_eq.q.id, 0,
1390                                                         0, 1, 1);
1391         }
1392         return ret;
1393 }
1394
1395 static void
1396 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1397               unsigned int num_sg, struct beiscsi_io_task *io_task)
1398 {
1399         struct iscsi_sge *psgl;
1400         unsigned short sg_len, index;
1401         unsigned int sge_len = 0;
1402         unsigned long long addr;
1403         struct scatterlist *l_sg;
1404         unsigned int offset;
1405
1406         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1407                                       io_task->bhs_pa.u.a32.address_lo);
1408         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1409                                       io_task->bhs_pa.u.a32.address_hi);
1410
1411         l_sg = sg;
1412         for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1413                 if (index == 0) {
1414                         sg_len = sg_dma_len(sg);
1415                         addr = (u64) sg_dma_address(sg);
1416                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1417                                                         (addr & 0xFFFFFFFF));
1418                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1419                                                         (addr >> 32));
1420                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1421                                                         sg_len);
1422                         sge_len = sg_len;
1423                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1424                                                         1);
1425                 } else {
1426                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1427                                                         0);
1428                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1429                                                         pwrb, sge_len);
1430                         sg_len = sg_dma_len(sg);
1431                         addr = (u64) sg_dma_address(sg);
1432                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1433                                                         (addr & 0xFFFFFFFF));
1434                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1435                                                         (addr >> 32));
1436                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1437                                                         sg_len);
1438                 }
1439         }
1440         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1441         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1442
1443         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1444
1445         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1446                         io_task->bhs_pa.u.a32.address_hi);
1447         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1448                         io_task->bhs_pa.u.a32.address_lo);
1449
1450         if (num_sg == 2)
1451                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1452         sg = l_sg;
1453         psgl++;
1454         psgl++;
1455         offset = 0;
1456         for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1457                 sg_len = sg_dma_len(sg);
1458                 addr = (u64) sg_dma_address(sg);
1459                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1460                                                 (addr & 0xFFFFFFFF));
1461                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1462                                                 (addr >> 32));
1463                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1464                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1465                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1466                 offset += sg_len;
1467         }
1468         psgl--;
1469         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1470 }
1471
1472 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1473 {
1474         struct iscsi_sge *psgl;
1475         unsigned long long addr;
1476         struct beiscsi_io_task *io_task = task->dd_data;
1477         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1478         struct beiscsi_hba *phba = beiscsi_conn->phba;
1479
1480         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1481         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1482                                 io_task->bhs_pa.u.a32.address_lo);
1483         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1484                                 io_task->bhs_pa.u.a32.address_hi);
1485
1486         if (task->data) {
1487                 if (task->data_count) {
1488                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1489                         addr = (u64) pci_map_single(phba->pcidev,
1490                                                     task->data,
1491                                                     task->data_count, 1);
1492                 } else {
1493                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1494                         addr = 0;
1495                 }
1496                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1497                                                 (addr & 0xFFFFFFFF));
1498                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1499                                                 (addr >> 32));
1500                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1501                                                 task->data_count);
1502
1503                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1504         } else {
1505                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1506                 addr = 0;
1507         }
1508
1509         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1510
1511         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1512
1513         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1514                       io_task->bhs_pa.u.a32.address_hi);
1515         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1516                       io_task->bhs_pa.u.a32.address_lo);
1517         if (task->data) {
1518                 psgl++;
1519                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1520                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1521                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1522                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1523                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1524                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1525
1526                 psgl++;
1527                 if (task->data) {
1528                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1529                                                 (addr & 0xFFFFFFFF));
1530                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1531                                                 (addr >> 32));
1532                 }
1533                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1534         }
1535         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1536 }
1537
1538 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1539 {
1540         unsigned int num_cq_pages, num_eq_pages, num_async_pdu_buf_pages;
1541         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1542         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1543
1544         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1545                                       sizeof(struct sol_cqe));
1546         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
1547                                       sizeof(struct be_eq_entry));
1548         num_async_pdu_buf_pages =
1549                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1550                                        phba->params.defpdu_hdr_sz);
1551         num_async_pdu_buf_sgl_pages =
1552                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1553                                        sizeof(struct phys_addr));
1554         num_async_pdu_data_pages =
1555                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1556                                        phba->params.defpdu_data_sz);
1557         num_async_pdu_data_sgl_pages =
1558                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1559                                        sizeof(struct phys_addr));
1560
1561         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1562
1563         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1564                                                  BE_ISCSI_PDU_HEADER_SIZE;
1565         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1566                                             sizeof(struct hwi_context_memory);
1567
1568         phba->mem_req[HWI_MEM_CQ] = num_cq_pages * PAGE_SIZE;
1569         phba->mem_req[HWI_MEM_EQ] = num_eq_pages * PAGE_SIZE;
1570
1571         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1572             * (phba->params.wrbs_per_cxn)
1573             * phba->params.cxns_per_ctrl;
1574         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1575                                  (phba->params.wrbs_per_cxn);
1576         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1577                                 phba->params.cxns_per_ctrl);
1578
1579         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1580                 phba->params.icds_per_ctrl;
1581         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1582                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1583
1584         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1585                 num_async_pdu_buf_pages * PAGE_SIZE;
1586         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1587                 num_async_pdu_data_pages * PAGE_SIZE;
1588         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1589                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1590         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1591                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1592         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1593                 phba->params.asyncpdus_per_ctrl *
1594                 sizeof(struct async_pdu_handle);
1595         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1596                 phba->params.asyncpdus_per_ctrl *
1597                 sizeof(struct async_pdu_handle);
1598         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1599                 sizeof(struct hwi_async_pdu_context) +
1600                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1601 }
1602
1603 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1604 {
1605         struct be_mem_descriptor *mem_descr;
1606         dma_addr_t bus_add;
1607         struct mem_array *mem_arr, *mem_arr_orig;
1608         unsigned int i, j, alloc_size, curr_alloc_size;
1609
1610         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1611         if (!phba->phwi_ctrlr)
1612                 return -ENOMEM;
1613
1614         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1615                                  GFP_KERNEL);
1616         if (!phba->init_mem) {
1617                 kfree(phba->phwi_ctrlr);
1618                 return -ENOMEM;
1619         }
1620
1621         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1622                                GFP_KERNEL);
1623         if (!mem_arr_orig) {
1624                 kfree(phba->init_mem);
1625                 kfree(phba->phwi_ctrlr);
1626                 return -ENOMEM;
1627         }
1628
1629         mem_descr = phba->init_mem;
1630         for (i = 0; i < SE_MEM_MAX; i++) {
1631                 j = 0;
1632                 mem_arr = mem_arr_orig;
1633                 alloc_size = phba->mem_req[i];
1634                 memset(mem_arr, 0, sizeof(struct mem_array) *
1635                        BEISCSI_MAX_FRAGS_INIT);
1636                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1637                 do {
1638                         mem_arr->virtual_address = pci_alloc_consistent(
1639                                                         phba->pcidev,
1640                                                         curr_alloc_size,
1641                                                         &bus_add);
1642                         if (!mem_arr->virtual_address) {
1643                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1644                                         goto free_mem;
1645                                 if (curr_alloc_size -
1646                                         rounddown_pow_of_two(curr_alloc_size))
1647                                         curr_alloc_size = rounddown_pow_of_two
1648                                                              (curr_alloc_size);
1649                                 else
1650                                         curr_alloc_size = curr_alloc_size / 2;
1651                         } else {
1652                                 mem_arr->bus_address.u.
1653                                     a64.address = (__u64) bus_add;
1654                                 mem_arr->size = curr_alloc_size;
1655                                 alloc_size -= curr_alloc_size;
1656                                 curr_alloc_size = min(be_max_phys_size *
1657                                                       1024, alloc_size);
1658                                 j++;
1659                                 mem_arr++;
1660                         }
1661                 } while (alloc_size);
1662                 mem_descr->num_elements = j;
1663                 mem_descr->size_in_bytes = phba->mem_req[i];
1664                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1665                                                GFP_KERNEL);
1666                 if (!mem_descr->mem_array)
1667                         goto free_mem;
1668
1669                 memcpy(mem_descr->mem_array, mem_arr_orig,
1670                        sizeof(struct mem_array) * j);
1671                 mem_descr++;
1672         }
1673         kfree(mem_arr_orig);
1674         return 0;
1675 free_mem:
1676         mem_descr->num_elements = j;
1677         while ((i) || (j)) {
1678                 for (j = mem_descr->num_elements; j > 0; j--) {
1679                         pci_free_consistent(phba->pcidev,
1680                                             mem_descr->mem_array[j - 1].size,
1681                                             mem_descr->mem_array[j - 1].
1682                                             virtual_address,
1683                                             mem_descr->mem_array[j - 1].
1684                                             bus_address.u.a64.address);
1685                 }
1686                 if (i) {
1687                         i--;
1688                         kfree(mem_descr->mem_array);
1689                         mem_descr--;
1690                 }
1691         }
1692         kfree(mem_arr_orig);
1693         kfree(phba->init_mem);
1694         kfree(phba->phwi_ctrlr);
1695         return -ENOMEM;
1696 }
1697
1698 static int beiscsi_get_memory(struct beiscsi_hba *phba)
1699 {
1700         beiscsi_find_mem_req(phba);
1701         return beiscsi_alloc_mem(phba);
1702 }
1703
1704 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1705 {
1706         struct pdu_data_out *pdata_out;
1707         struct pdu_nop_out *pnop_out;
1708         struct be_mem_descriptor *mem_descr;
1709
1710         mem_descr = phba->init_mem;
1711         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1712         pdata_out =
1713             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1714         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1715
1716         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1717                       IIOC_SCSI_DATA);
1718
1719         pnop_out =
1720             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1721                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1722
1723         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1724         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1725         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1726         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1727 }
1728
1729 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1730 {
1731         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1732         struct wrb_handle *pwrb_handle;
1733         struct hwi_controller *phwi_ctrlr;
1734         struct hwi_wrb_context *pwrb_context;
1735         struct iscsi_wrb *pwrb;
1736         unsigned int num_cxn_wrbh;
1737         unsigned int num_cxn_wrb, j, idx, index;
1738
1739         mem_descr_wrbh = phba->init_mem;
1740         mem_descr_wrbh += HWI_MEM_WRBH;
1741
1742         mem_descr_wrb = phba->init_mem;
1743         mem_descr_wrb += HWI_MEM_WRB;
1744
1745         idx = 0;
1746         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
1747         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
1748                         ((sizeof(struct wrb_handle)) *
1749                          phba->params.wrbs_per_cxn));
1750         phwi_ctrlr = phba->phwi_ctrlr;
1751
1752         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
1753                 pwrb_context = &phwi_ctrlr->wrb_context[index];
1754                 SE_DEBUG(DBG_LVL_8, "cid=%d pwrb_context=%p \n", index,
1755                                                 pwrb_context);
1756                 pwrb_context->pwrb_handle_base =
1757                                 kzalloc(sizeof(struct wrb_handle *) *
1758                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
1759                 pwrb_context->pwrb_handle_basestd =
1760                                 kzalloc(sizeof(struct wrb_handle *) *
1761                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
1762                 if (num_cxn_wrbh) {
1763                         pwrb_context->alloc_index = 0;
1764                         pwrb_context->wrb_handles_available = 0;
1765                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1766                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1767                                 pwrb_context->pwrb_handle_basestd[j] =
1768                                                                 pwrb_handle;
1769                                 pwrb_context->wrb_handles_available++;
1770                                 pwrb_handle++;
1771                         }
1772                         pwrb_context->free_index = 0;
1773                         num_cxn_wrbh--;
1774                 } else {
1775                         idx++;
1776                         pwrb_handle =
1777                             mem_descr_wrbh->mem_array[idx].virtual_address;
1778                         num_cxn_wrbh =
1779                             ((mem_descr_wrbh->mem_array[idx].size) /
1780                              ((sizeof(struct wrb_handle)) *
1781                               phba->params.wrbs_per_cxn));
1782                         pwrb_context->alloc_index = 0;
1783                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1784                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1785                                 pwrb_context->pwrb_handle_basestd[j] =
1786                                     pwrb_handle;
1787                                 pwrb_context->wrb_handles_available++;
1788                                 pwrb_handle++;
1789                         }
1790                         pwrb_context->free_index = 0;
1791                         num_cxn_wrbh--;
1792                 }
1793         }
1794         idx = 0;
1795         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1796         num_cxn_wrb =
1797             ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
1798              phba->params.wrbs_per_cxn);
1799
1800         for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
1801                 pwrb_context = &phwi_ctrlr->wrb_context[index];
1802                 if (num_cxn_wrb) {
1803                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1804                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1805                                 pwrb_handle->pwrb = pwrb;
1806                                 pwrb++;
1807                         }
1808                         num_cxn_wrb--;
1809                 } else {
1810                         idx++;
1811                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1812                         num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
1813                                         (sizeof(struct iscsi_wrb)) *
1814                                         phba->params.wrbs_per_cxn);
1815                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1816                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1817                                 pwrb_handle->pwrb = pwrb;
1818                                 pwrb++;
1819                         }
1820                         num_cxn_wrb--;
1821                 }
1822         }
1823 }
1824
1825 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
1826 {
1827         struct hwi_controller *phwi_ctrlr;
1828         struct hba_parameters *p = &phba->params;
1829         struct hwi_async_pdu_context *pasync_ctx;
1830         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
1831         unsigned int index;
1832         struct be_mem_descriptor *mem_descr;
1833
1834         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1835         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
1836
1837         phwi_ctrlr = phba->phwi_ctrlr;
1838         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
1839                                 mem_descr->mem_array[0].virtual_address;
1840         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
1841         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
1842
1843         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
1844         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
1845         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
1846         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
1847
1848         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1849         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
1850         if (mem_descr->mem_array[0].virtual_address) {
1851                 SE_DEBUG(DBG_LVL_8,
1852                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
1853                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1854         } else
1855                 shost_printk(KERN_WARNING, phba->shost,
1856                              "No Virtual address \n");
1857
1858         pasync_ctx->async_header.va_base =
1859                         mem_descr->mem_array[0].virtual_address;
1860
1861         pasync_ctx->async_header.pa_base.u.a64.address =
1862                         mem_descr->mem_array[0].bus_address.u.a64.address;
1863
1864         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1865         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
1866         if (mem_descr->mem_array[0].virtual_address) {
1867                 SE_DEBUG(DBG_LVL_8,
1868                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
1869                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1870         } else
1871                 shost_printk(KERN_WARNING, phba->shost,
1872                             "No Virtual address \n");
1873         pasync_ctx->async_header.ring_base =
1874                         mem_descr->mem_array[0].virtual_address;
1875
1876         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1877         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
1878         if (mem_descr->mem_array[0].virtual_address) {
1879                 SE_DEBUG(DBG_LVL_8,
1880                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
1881                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1882         } else
1883                 shost_printk(KERN_WARNING, phba->shost,
1884                             "No Virtual address \n");
1885
1886         pasync_ctx->async_header.handle_base =
1887                         mem_descr->mem_array[0].virtual_address;
1888         pasync_ctx->async_header.writables = 0;
1889         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
1890
1891         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1892         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
1893         if (mem_descr->mem_array[0].virtual_address) {
1894                 SE_DEBUG(DBG_LVL_8,
1895                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
1896                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1897         } else
1898                 shost_printk(KERN_WARNING, phba->shost,
1899                             "No Virtual address \n");
1900         pasync_ctx->async_data.va_base =
1901                         mem_descr->mem_array[0].virtual_address;
1902         pasync_ctx->async_data.pa_base.u.a64.address =
1903                         mem_descr->mem_array[0].bus_address.u.a64.address;
1904
1905         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1906         mem_descr += HWI_MEM_ASYNC_DATA_RING;
1907         if (mem_descr->mem_array[0].virtual_address) {
1908                 SE_DEBUG(DBG_LVL_8,
1909                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
1910                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1911         } else
1912                 shost_printk(KERN_WARNING, phba->shost,
1913                              "No Virtual address \n");
1914
1915         pasync_ctx->async_data.ring_base =
1916                         mem_descr->mem_array[0].virtual_address;
1917
1918         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1919         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
1920         if (!mem_descr->mem_array[0].virtual_address)
1921                 shost_printk(KERN_WARNING, phba->shost,
1922                             "No Virtual address \n");
1923
1924         pasync_ctx->async_data.handle_base =
1925                         mem_descr->mem_array[0].virtual_address;
1926         pasync_ctx->async_data.writables = 0;
1927         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
1928
1929         pasync_header_h =
1930                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
1931         pasync_data_h =
1932                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
1933
1934         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
1935                 pasync_header_h->cri = -1;
1936                 pasync_header_h->index = (char)index;
1937                 INIT_LIST_HEAD(&pasync_header_h->link);
1938                 pasync_header_h->pbuffer =
1939                         (void *)((unsigned long)
1940                         (pasync_ctx->async_header.va_base) +
1941                         (p->defpdu_hdr_sz * index));
1942
1943                 pasync_header_h->pa.u.a64.address =
1944                         pasync_ctx->async_header.pa_base.u.a64.address +
1945                         (p->defpdu_hdr_sz * index);
1946
1947                 list_add_tail(&pasync_header_h->link,
1948                                 &pasync_ctx->async_header.free_list);
1949                 pasync_header_h++;
1950                 pasync_ctx->async_header.free_entries++;
1951                 pasync_ctx->async_header.writables++;
1952
1953                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
1954                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
1955                                header_busy_list);
1956                 pasync_data_h->cri = -1;
1957                 pasync_data_h->index = (char)index;
1958                 INIT_LIST_HEAD(&pasync_data_h->link);
1959                 pasync_data_h->pbuffer =
1960                         (void *)((unsigned long)
1961                         (pasync_ctx->async_data.va_base) +
1962                         (p->defpdu_data_sz * index));
1963
1964                 pasync_data_h->pa.u.a64.address =
1965                     pasync_ctx->async_data.pa_base.u.a64.address +
1966                     (p->defpdu_data_sz * index);
1967
1968                 list_add_tail(&pasync_data_h->link,
1969                               &pasync_ctx->async_data.free_list);
1970                 pasync_data_h++;
1971                 pasync_ctx->async_data.free_entries++;
1972                 pasync_ctx->async_data.writables++;
1973
1974                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
1975         }
1976
1977         pasync_ctx->async_header.host_write_ptr = 0;
1978         pasync_ctx->async_header.ep_read_ptr = -1;
1979         pasync_ctx->async_data.host_write_ptr = 0;
1980         pasync_ctx->async_data.ep_read_ptr = -1;
1981 }
1982
1983 static int
1984 be_sgl_create_contiguous(void *virtual_address,
1985                          u64 physical_address, u32 length,
1986                          struct be_dma_mem *sgl)
1987 {
1988         WARN_ON(!virtual_address);
1989         WARN_ON(!physical_address);
1990         WARN_ON(!length > 0);
1991         WARN_ON(!sgl);
1992
1993         sgl->va = virtual_address;
1994         sgl->dma = physical_address;
1995         sgl->size = length;
1996
1997         return 0;
1998 }
1999
2000 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2001 {
2002         memset(sgl, 0, sizeof(*sgl));
2003 }
2004
2005 static void
2006 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2007                      struct mem_array *pmem, struct be_dma_mem *sgl)
2008 {
2009         if (sgl->va)
2010                 be_sgl_destroy_contiguous(sgl);
2011
2012         be_sgl_create_contiguous(pmem->virtual_address,
2013                                  pmem->bus_address.u.a64.address,
2014                                  pmem->size, sgl);
2015 }
2016
2017 static void
2018 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2019                            struct mem_array *pmem, struct be_dma_mem *sgl)
2020 {
2021         if (sgl->va)
2022                 be_sgl_destroy_contiguous(sgl);
2023
2024         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2025                                  pmem->bus_address.u.a64.address,
2026                                  pmem->size, sgl);
2027 }
2028
2029 static int be_fill_queue(struct be_queue_info *q,
2030                 u16 len, u16 entry_size, void *vaddress)
2031 {
2032         struct be_dma_mem *mem = &q->dma_mem;
2033
2034         memset(q, 0, sizeof(*q));
2035         q->len = len;
2036         q->entry_size = entry_size;
2037         mem->size = len * entry_size;
2038         mem->va = vaddress;
2039         if (!mem->va)
2040                 return -ENOMEM;
2041         memset(mem->va, 0, mem->size);
2042         return 0;
2043 }
2044
2045 static int beiscsi_create_eq(struct beiscsi_hba *phba,
2046                              struct hwi_context_memory *phwi_context)
2047 {
2048         unsigned int idx;
2049         int ret;
2050         struct be_queue_info *eq;
2051         struct be_dma_mem *mem;
2052         struct be_mem_descriptor *mem_descr;
2053         void *eq_vaddress;
2054
2055         idx = 0;
2056         eq = &phwi_context->be_eq.q;
2057         mem = &eq->dma_mem;
2058         mem_descr = phba->init_mem;
2059         mem_descr += HWI_MEM_EQ;
2060         eq_vaddress = mem_descr->mem_array[idx].virtual_address;
2061
2062         ret = be_fill_queue(eq, phba->params.num_eq_entries,
2063                             sizeof(struct be_eq_entry), eq_vaddress);
2064         if (ret) {
2065                 shost_printk(KERN_ERR, phba->shost,
2066                              "be_fill_queue Failed for EQ \n");
2067                 return ret;
2068         }
2069
2070         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2071
2072         ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2073                                     phwi_context->be_eq.cur_eqd);
2074         if (ret) {
2075                 shost_printk(KERN_ERR, phba->shost, "beiscsi_cmd_eq_create"
2076                              "Failedfor EQ \n");
2077                 return ret;
2078         }
2079         SE_DEBUG(DBG_LVL_8, "eq id is %d\n", phwi_context->be_eq.q.id);
2080         return 0;
2081 }
2082
2083 static int beiscsi_create_cq(struct beiscsi_hba *phba,
2084                              struct hwi_context_memory *phwi_context)
2085 {
2086         unsigned int idx;
2087         int ret;
2088         struct be_queue_info *cq, *eq;
2089         struct be_dma_mem *mem;
2090         struct be_mem_descriptor *mem_descr;
2091         void *cq_vaddress;
2092
2093         idx = 0;
2094         cq = &phwi_context->be_cq;
2095         eq = &phwi_context->be_eq.q;
2096         mem = &cq->dma_mem;
2097         mem_descr = phba->init_mem;
2098         mem_descr += HWI_MEM_CQ;
2099         cq_vaddress = mem_descr->mem_array[idx].virtual_address;
2100         ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2101                             sizeof(struct sol_cqe), cq_vaddress);
2102         if (ret) {
2103                 shost_printk(KERN_ERR, phba->shost,
2104                              "be_fill_queue Failed for ISCSI CQ \n");
2105                 return ret;
2106         }
2107
2108         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2109         ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, false, 0);
2110         if (ret) {
2111                 shost_printk(KERN_ERR, phba->shost,
2112                              "beiscsi_cmd_eq_create Failed for ISCSI CQ \n");
2113                 return ret;
2114         }
2115         SE_DEBUG(DBG_LVL_8, "iscsi cq id is %d\n", phwi_context->be_cq.id);
2116         SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2117         return 0;
2118 }
2119
2120 static int
2121 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2122                        struct hwi_context_memory *phwi_context,
2123                        struct hwi_controller *phwi_ctrlr,
2124                        unsigned int def_pdu_ring_sz)
2125 {
2126         unsigned int idx;
2127         int ret;
2128         struct be_queue_info *dq, *cq;
2129         struct be_dma_mem *mem;
2130         struct be_mem_descriptor *mem_descr;
2131         void *dq_vaddress;
2132
2133         idx = 0;
2134         dq = &phwi_context->be_def_hdrq;
2135         cq = &phwi_context->be_cq;
2136         mem = &dq->dma_mem;
2137         mem_descr = phba->init_mem;
2138         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2139         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2140         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2141                             sizeof(struct phys_addr),
2142                             sizeof(struct phys_addr), dq_vaddress);
2143         if (ret) {
2144                 shost_printk(KERN_ERR, phba->shost,
2145                              "be_fill_queue Failed for DEF PDU HDR\n");
2146                 return ret;
2147         }
2148         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2149         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2150                                               def_pdu_ring_sz,
2151                                               phba->params.defpdu_hdr_sz);
2152         if (ret) {
2153                 shost_printk(KERN_ERR, phba->shost,
2154                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2155                 return ret;
2156         }
2157         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2158         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2159                  phwi_context->be_def_hdrq.id);
2160         hwi_post_async_buffers(phba, 1);
2161         return 0;
2162 }
2163
2164 static int
2165 beiscsi_create_def_data(struct beiscsi_hba *phba,
2166                         struct hwi_context_memory *phwi_context,
2167                         struct hwi_controller *phwi_ctrlr,
2168                         unsigned int def_pdu_ring_sz)
2169 {
2170         unsigned int idx;
2171         int ret;
2172         struct be_queue_info *dataq, *cq;
2173         struct be_dma_mem *mem;
2174         struct be_mem_descriptor *mem_descr;
2175         void *dq_vaddress;
2176
2177         idx = 0;
2178         dataq = &phwi_context->be_def_dataq;
2179         cq = &phwi_context->be_cq;
2180         mem = &dataq->dma_mem;
2181         mem_descr = phba->init_mem;
2182         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2183         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2184         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2185                             sizeof(struct phys_addr),
2186                             sizeof(struct phys_addr), dq_vaddress);
2187         if (ret) {
2188                 shost_printk(KERN_ERR, phba->shost,
2189                              "be_fill_queue Failed for DEF PDU DATA\n");
2190                 return ret;
2191         }
2192         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2193         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2194                                               def_pdu_ring_sz,
2195                                               phba->params.defpdu_data_sz);
2196         if (ret) {
2197                 shost_printk(KERN_ERR, phba->shost,
2198                              "be_cmd_create_default_pdu_queue Failed"
2199                              " for DEF PDU DATA\n");
2200                 return ret;
2201         }
2202         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2203         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2204                  phwi_context->be_def_dataq.id);
2205         hwi_post_async_buffers(phba, 0);
2206         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2207         return 0;
2208 }
2209
2210 static int
2211 beiscsi_post_pages(struct beiscsi_hba *phba)
2212 {
2213         struct be_mem_descriptor *mem_descr;
2214         struct mem_array *pm_arr;
2215         unsigned int page_offset, i;
2216         struct be_dma_mem sgl;
2217         int status;
2218
2219         mem_descr = phba->init_mem;
2220         mem_descr += HWI_MEM_SGE;
2221         pm_arr = mem_descr->mem_array;
2222
2223         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2224                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2225         for (i = 0; i < mem_descr->num_elements; i++) {
2226                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2227                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2228                                                 page_offset,
2229                                                 (pm_arr->size / PAGE_SIZE));
2230                 page_offset += pm_arr->size / PAGE_SIZE;
2231                 if (status != 0) {
2232                         shost_printk(KERN_ERR, phba->shost,
2233                                      "post sgl failed.\n");
2234                         return status;
2235                 }
2236                 pm_arr++;
2237         }
2238         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2239         return 0;
2240 }
2241
2242 static int
2243 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2244                          struct hwi_context_memory *phwi_context,
2245                          struct hwi_controller *phwi_ctrlr)
2246 {
2247         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2248         u64 pa_addr_lo;
2249         unsigned int idx, num, i;
2250         struct mem_array *pwrb_arr;
2251         void *wrb_vaddr;
2252         struct be_dma_mem sgl;
2253         struct be_mem_descriptor *mem_descr;
2254         int status;
2255
2256         idx = 0;
2257         mem_descr = phba->init_mem;
2258         mem_descr += HWI_MEM_WRB;
2259         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2260                            GFP_KERNEL);
2261         if (!pwrb_arr) {
2262                 shost_printk(KERN_ERR, phba->shost,
2263                              "Memory alloc failed in create wrb ring.\n");
2264                 return -ENOMEM;
2265         }
2266         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2267         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2268         num_wrb_rings = mem_descr->mem_array[idx].size /
2269                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2270
2271         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2272                 if (num_wrb_rings) {
2273                         pwrb_arr[num].virtual_address = wrb_vaddr;
2274                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2275                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2276                                             sizeof(struct iscsi_wrb);
2277                         wrb_vaddr += pwrb_arr[num].size;
2278                         pa_addr_lo += pwrb_arr[num].size;
2279                         num_wrb_rings--;
2280                 } else {
2281                         idx++;
2282                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2283                         pa_addr_lo = mem_descr->mem_array[idx].\
2284                                         bus_address.u.a64.address;
2285                         num_wrb_rings = mem_descr->mem_array[idx].size /
2286                                         (phba->params.wrbs_per_cxn *
2287                                         sizeof(struct iscsi_wrb));
2288                         pwrb_arr[num].virtual_address = wrb_vaddr;
2289                         pwrb_arr[num].bus_address.u.a64.address\
2290                                                 = pa_addr_lo;
2291                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2292                                                  sizeof(struct iscsi_wrb);
2293                         wrb_vaddr += pwrb_arr[num].size;
2294                         pa_addr_lo   += pwrb_arr[num].size;
2295                         num_wrb_rings--;
2296                 }
2297         }
2298         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2299                 wrb_mem_index = 0;
2300                 offset = 0;
2301                 size = 0;
2302
2303                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2304                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2305                                             &phwi_context->be_wrbq[i]);
2306                 if (status != 0) {
2307                         shost_printk(KERN_ERR, phba->shost,
2308                                      "wrbq create failed.");
2309                         return status;
2310                 }
2311                 phwi_ctrlr->wrb_context[i].cid = phwi_context->be_wrbq[i].id;
2312         }
2313         kfree(pwrb_arr);
2314         return 0;
2315 }
2316
2317 static void free_wrb_handles(struct beiscsi_hba *phba)
2318 {
2319         unsigned int index;
2320         struct hwi_controller *phwi_ctrlr;
2321         struct hwi_wrb_context *pwrb_context;
2322
2323         phwi_ctrlr = phba->phwi_ctrlr;
2324         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2325                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2326                 kfree(pwrb_context->pwrb_handle_base);
2327                 kfree(pwrb_context->pwrb_handle_basestd);
2328         }
2329 }
2330
2331 static void hwi_cleanup(struct beiscsi_hba *phba)
2332 {
2333         struct be_queue_info *q;
2334         struct be_ctrl_info *ctrl = &phba->ctrl;
2335         struct hwi_controller *phwi_ctrlr;
2336         struct hwi_context_memory *phwi_context;
2337         int i;
2338
2339         phwi_ctrlr = phba->phwi_ctrlr;
2340         phwi_context = phwi_ctrlr->phwi_ctxt;
2341         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2342                 q = &phwi_context->be_wrbq[i];
2343                 if (q->created)
2344                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2345         }
2346
2347         free_wrb_handles(phba);
2348
2349         q = &phwi_context->be_def_hdrq;
2350         if (q->created)
2351                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2352
2353         q = &phwi_context->be_def_dataq;
2354         if (q->created)
2355                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2356
2357         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2358
2359         q = &phwi_context->be_cq;
2360         if (q->created)
2361                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2362
2363         q = &phwi_context->be_eq.q;
2364         if (q->created)
2365                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2366 }
2367
2368 static int hwi_init_port(struct beiscsi_hba *phba)
2369 {
2370         struct hwi_controller *phwi_ctrlr;
2371         struct hwi_context_memory *phwi_context;
2372         unsigned int def_pdu_ring_sz;
2373         struct be_ctrl_info *ctrl = &phba->ctrl;
2374         int status;
2375
2376         def_pdu_ring_sz =
2377                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2378         phwi_ctrlr = phba->phwi_ctrlr;
2379
2380         phwi_context = phwi_ctrlr->phwi_ctxt;
2381         phwi_context->be_eq.max_eqd = 0;
2382         phwi_context->be_eq.min_eqd = 0;
2383         phwi_context->be_eq.cur_eqd = 64;
2384         phwi_context->be_eq.enable_aic = false;
2385         be_cmd_fw_initialize(&phba->ctrl);
2386         status = beiscsi_create_eq(phba, phwi_context);
2387         if (status != 0) {
2388                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2389                 goto error;
2390         }
2391
2392         status = mgmt_check_supported_fw(ctrl);
2393         if (status != 0) {
2394                 shost_printk(KERN_ERR, phba->shost,
2395                              "Unsupported fw version \n");
2396                 goto error;
2397         }
2398
2399         status = mgmt_get_fw_config(ctrl, phba);
2400         if (status != 0) {
2401                 shost_printk(KERN_ERR, phba->shost,
2402                              "Error getting fw config\n");
2403                 goto error;
2404         }
2405
2406         status = beiscsi_create_cq(phba, phwi_context);
2407         if (status != 0) {
2408                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2409                 goto error;
2410         }
2411
2412         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2413                                         def_pdu_ring_sz);
2414         if (status != 0) {
2415                 shost_printk(KERN_ERR, phba->shost,
2416                              "Default Header not created\n");
2417                 goto error;
2418         }
2419
2420         status = beiscsi_create_def_data(phba, phwi_context,
2421                                          phwi_ctrlr, def_pdu_ring_sz);
2422         if (status != 0) {
2423                 shost_printk(KERN_ERR, phba->shost,
2424                              "Default Data not created\n");
2425                 goto error;
2426         }
2427
2428         status = beiscsi_post_pages(phba);
2429         if (status != 0) {
2430                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2431                 goto error;
2432         }
2433
2434         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2435         if (status != 0) {
2436                 shost_printk(KERN_ERR, phba->shost,
2437                              "WRB Rings not created\n");
2438                 goto error;
2439         }
2440
2441         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2442         return 0;
2443
2444 error:
2445         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2446         hwi_cleanup(phba);
2447         return -ENOMEM;
2448 }
2449
2450
2451 static int hwi_init_controller(struct beiscsi_hba *phba)
2452 {
2453         struct hwi_controller *phwi_ctrlr;
2454
2455         phwi_ctrlr = phba->phwi_ctrlr;
2456         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2457                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2458                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2459                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2460                          phwi_ctrlr->phwi_ctxt);
2461         } else {
2462                 shost_printk(KERN_ERR, phba->shost,
2463                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2464                              "Failing to load\n");
2465                 return -ENOMEM;
2466         }
2467
2468         iscsi_init_global_templates(phba);
2469         beiscsi_init_wrb_handle(phba);
2470         hwi_init_async_pdu_ctx(phba);
2471         if (hwi_init_port(phba) != 0) {
2472                 shost_printk(KERN_ERR, phba->shost,
2473                              "hwi_init_controller failed\n");
2474                 return -ENOMEM;
2475         }
2476         return 0;
2477 }
2478
2479 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2480 {
2481         struct be_mem_descriptor *mem_descr;
2482         int i, j;
2483
2484         mem_descr = phba->init_mem;
2485         i = 0;
2486         j = 0;
2487         for (i = 0; i < SE_MEM_MAX; i++) {
2488                 for (j = mem_descr->num_elements; j > 0; j--) {
2489                         pci_free_consistent(phba->pcidev,
2490                           mem_descr->mem_array[j - 1].size,
2491                           mem_descr->mem_array[j - 1].virtual_address,
2492                           mem_descr->mem_array[j - 1].bus_address.
2493                                 u.a64.address);
2494                 }
2495                 kfree(mem_descr->mem_array);
2496                 mem_descr++;
2497         }
2498         kfree(phba->init_mem);
2499         kfree(phba->phwi_ctrlr);
2500 }
2501
2502 static int beiscsi_init_controller(struct beiscsi_hba *phba)
2503 {
2504         int ret = -ENOMEM;
2505
2506         ret = beiscsi_get_memory(phba);
2507         if (ret < 0) {
2508                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2509                              "Failed in beiscsi_alloc_memory \n");
2510                 return ret;
2511         }
2512
2513         ret = hwi_init_controller(phba);
2514         if (ret)
2515                 goto free_init;
2516         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2517         return 0;
2518
2519 free_init:
2520         beiscsi_free_mem(phba);
2521         return -ENOMEM;
2522 }
2523
2524 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2525 {
2526         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2527         struct sgl_handle *psgl_handle;
2528         struct iscsi_sge *pfrag;
2529         unsigned int arr_index, i, idx;
2530
2531         phba->io_sgl_hndl_avbl = 0;
2532         phba->eh_sgl_hndl_avbl = 0;
2533         mem_descr_sglh = phba->init_mem;
2534         mem_descr_sglh += HWI_MEM_SGLH;
2535         if (1 == mem_descr_sglh->num_elements) {
2536                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2537                                                  phba->params.ios_per_ctrl,
2538                                                  GFP_KERNEL);
2539                 if (!phba->io_sgl_hndl_base) {
2540                         shost_printk(KERN_ERR, phba->shost,
2541                                      "Mem Alloc Failed. Failing to load\n");
2542                         return -ENOMEM;
2543                 }
2544                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2545                                                  (phba->params.icds_per_ctrl -
2546                                                  phba->params.ios_per_ctrl),
2547                                                  GFP_KERNEL);
2548                 if (!phba->eh_sgl_hndl_base) {
2549                         kfree(phba->io_sgl_hndl_base);
2550                         shost_printk(KERN_ERR, phba->shost,
2551                                      "Mem Alloc Failed. Failing to load\n");
2552                         return -ENOMEM;
2553                 }
2554         } else {
2555                 shost_printk(KERN_ERR, phba->shost,
2556                              "HWI_MEM_SGLH is more than one element."
2557                              "Failing to load\n");
2558                 return -ENOMEM;
2559         }
2560
2561         arr_index = 0;
2562         idx = 0;
2563         while (idx < mem_descr_sglh->num_elements) {
2564                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2565
2566                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2567                       sizeof(struct sgl_handle)); i++) {
2568                         if (arr_index < phba->params.ios_per_ctrl) {
2569                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2570                                 phba->io_sgl_hndl_avbl++;
2571                                 arr_index++;
2572                         } else {
2573                                 phba->eh_sgl_hndl_base[arr_index -
2574                                         phba->params.ios_per_ctrl] =
2575                                                                 psgl_handle;
2576                                 arr_index++;
2577                                 phba->eh_sgl_hndl_avbl++;
2578                         }
2579                         psgl_handle++;
2580                 }
2581                 idx++;
2582         }
2583         SE_DEBUG(DBG_LVL_8,
2584                  "phba->io_sgl_hndl_avbl=%d"
2585                  "phba->eh_sgl_hndl_avbl=%d \n",
2586                  phba->io_sgl_hndl_avbl,
2587                  phba->eh_sgl_hndl_avbl);
2588         mem_descr_sg = phba->init_mem;
2589         mem_descr_sg += HWI_MEM_SGE;
2590         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
2591                  mem_descr_sg->num_elements);
2592         arr_index = 0;
2593         idx = 0;
2594         while (idx < mem_descr_sg->num_elements) {
2595                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
2596
2597                 for (i = 0;
2598                      i < (mem_descr_sg->mem_array[idx].size) /
2599                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
2600                      i++) {
2601                         if (arr_index < phba->params.ios_per_ctrl)
2602                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
2603                         else
2604                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
2605                                                 phba->params.ios_per_ctrl];
2606                         psgl_handle->pfrag = pfrag;
2607                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
2608                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
2609                         pfrag += phba->params.num_sge_per_io;
2610                         psgl_handle->sgl_index =
2611                                 phba->fw_config.iscsi_cid_start + arr_index++;
2612                 }
2613                 idx++;
2614         }
2615         phba->io_sgl_free_index = 0;
2616         phba->io_sgl_alloc_index = 0;
2617         phba->eh_sgl_free_index = 0;
2618         phba->eh_sgl_alloc_index = 0;
2619         return 0;
2620 }
2621
2622 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
2623 {
2624         int i, new_cid;
2625
2626         phba->cid_array = kmalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
2627                                   GFP_KERNEL);
2628         if (!phba->cid_array) {
2629                 shost_printk(KERN_ERR, phba->shost,
2630                              "Failed to allocate memory in "
2631                              "hba_setup_cid_tbls\n");
2632                 return -ENOMEM;
2633         }
2634         phba->ep_array = kmalloc(sizeof(struct iscsi_endpoint *) *
2635                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
2636         if (!phba->ep_array) {
2637                 shost_printk(KERN_ERR, phba->shost,
2638                              "Failed to allocate memory in "
2639                              "hba_setup_cid_tbls \n");
2640                 kfree(phba->cid_array);
2641                 return -ENOMEM;
2642         }
2643         new_cid = phba->fw_config.iscsi_icd_start;
2644         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2645                 phba->cid_array[i] = new_cid;
2646                 new_cid += 2;
2647         }
2648         phba->avlbl_cids = phba->params.cxns_per_ctrl;
2649         return 0;
2650 }
2651
2652 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
2653 {
2654         struct be_ctrl_info *ctrl = &phba->ctrl;
2655         struct hwi_controller *phwi_ctrlr;
2656         struct hwi_context_memory *phwi_context;
2657         struct be_queue_info *eq;
2658         u8 __iomem *addr;
2659         u32 reg;
2660         u32 enabled;
2661
2662         phwi_ctrlr = phba->phwi_ctrlr;
2663         phwi_context = phwi_ctrlr->phwi_ctxt;
2664
2665         eq = &phwi_context->be_eq.q;
2666         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
2667                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
2668         reg = ioread32(addr);
2669         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
2670
2671         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2672         if (!enabled) {
2673                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2674                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
2675                 iowrite32(reg, addr);
2676                 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
2677
2678                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
2679         } else
2680                 shost_printk(KERN_WARNING, phba->shost,
2681                              "In hwi_enable_intr, Not Enabled \n");
2682         return true;
2683 }
2684
2685 static void hwi_disable_intr(struct beiscsi_hba *phba)
2686 {
2687         struct be_ctrl_info *ctrl = &phba->ctrl;
2688
2689         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
2690         u32 reg = ioread32(addr);
2691
2692         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2693         if (enabled) {
2694                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2695                 iowrite32(reg, addr);
2696         } else
2697                 shost_printk(KERN_WARNING, phba->shost,
2698                              "In hwi_disable_intr, Already Disabled \n");
2699 }
2700
2701 static int beiscsi_init_port(struct beiscsi_hba *phba)
2702 {
2703         int ret;
2704
2705         ret = beiscsi_init_controller(phba);
2706         if (ret < 0) {
2707                 shost_printk(KERN_ERR, phba->shost,
2708                              "beiscsi_dev_probe - Failed in"
2709                              "beiscsi_init_controller \n");
2710                 return ret;
2711         }
2712         ret = beiscsi_init_sgl_handle(phba);
2713         if (ret < 0) {
2714                 shost_printk(KERN_ERR, phba->shost,
2715                              "beiscsi_dev_probe - Failed in"
2716                              "beiscsi_init_sgl_handle \n");
2717                 goto do_cleanup_ctrlr;
2718         }
2719
2720         if (hba_setup_cid_tbls(phba)) {
2721                 shost_printk(KERN_ERR, phba->shost,
2722                              "Failed in hba_setup_cid_tbls\n");
2723                 kfree(phba->io_sgl_hndl_base);
2724                 kfree(phba->eh_sgl_hndl_base);
2725                 goto do_cleanup_ctrlr;
2726         }
2727
2728         return ret;
2729
2730 do_cleanup_ctrlr:
2731         hwi_cleanup(phba);
2732         return ret;
2733 }
2734
2735 static void hwi_purge_eq(struct beiscsi_hba *phba)
2736 {
2737         struct hwi_controller *phwi_ctrlr;
2738         struct hwi_context_memory *phwi_context;
2739         struct be_queue_info *eq;
2740         struct be_eq_entry *eqe = NULL;
2741
2742         phwi_ctrlr = phba->phwi_ctrlr;
2743         phwi_context = phwi_ctrlr->phwi_ctxt;
2744         eq = &phwi_context->be_eq.q;
2745         eqe = queue_tail_node(eq);
2746
2747         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
2748                                                 & EQE_VALID_MASK) {
2749                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2750                 queue_tail_inc(eq);
2751                 eqe = queue_tail_node(eq);
2752         }
2753 }
2754
2755 static void beiscsi_clean_port(struct beiscsi_hba *phba)
2756 {
2757         unsigned char mgmt_status;
2758
2759         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
2760         if (mgmt_status)
2761                 shost_printk(KERN_WARNING, phba->shost,
2762                              "mgmt_epfw_cleanup FAILED \n");
2763         hwi_cleanup(phba);
2764         hwi_purge_eq(phba);
2765         kfree(phba->io_sgl_hndl_base);
2766         kfree(phba->eh_sgl_hndl_base);
2767         kfree(phba->cid_array);
2768         kfree(phba->ep_array);
2769 }
2770
2771 void
2772 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
2773                            struct beiscsi_offload_params *params)
2774 {
2775         struct wrb_handle *pwrb_handle;
2776         struct iscsi_target_context_update_wrb *pwrb = NULL;
2777         struct be_mem_descriptor *mem_descr;
2778         struct beiscsi_hba *phba = beiscsi_conn->phba;
2779         u32 doorbell = 0;
2780
2781         /*
2782          * We can always use 0 here because it is reserved by libiscsi for
2783          * login/startup related tasks.
2784          */
2785         pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid, 0);
2786         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
2787         memset(pwrb, 0, sizeof(*pwrb));
2788         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2789                       max_burst_length, pwrb, params->dw[offsetof
2790                       (struct amap_beiscsi_offload_params,
2791                       max_burst_length) / 32]);
2792         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2793                       max_send_data_segment_length, pwrb,
2794                       params->dw[offsetof(struct amap_beiscsi_offload_params,
2795                       max_send_data_segment_length) / 32]);
2796         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2797                       first_burst_length,
2798                       pwrb,
2799                       params->dw[offsetof(struct amap_beiscsi_offload_params,
2800                       first_burst_length) / 32]);
2801
2802         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
2803                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2804                       erl) / 32] & OFFLD_PARAMS_ERL));
2805         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
2806                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2807                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
2808         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
2809                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2810                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
2811         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
2812                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2813                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
2814         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
2815                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2816                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
2817         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
2818                       pwrb,
2819                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2820                       exp_statsn) / 32] + 1));
2821         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
2822                       0x7);
2823         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
2824                       pwrb, pwrb_handle->wrb_index);
2825         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
2826                       pwrb, pwrb_handle->nxt_wrb_index);
2827         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2828                         session_state, pwrb, 0);
2829         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
2830                       pwrb, 1);
2831         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
2832                       pwrb, 0);
2833         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
2834                       0);
2835
2836         mem_descr = phba->init_mem;
2837         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2838
2839         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2840                         pad_buffer_addr_hi, pwrb,
2841                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
2842         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2843                         pad_buffer_addr_lo, pwrb,
2844                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
2845
2846         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
2847
2848         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
2849         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) <<
2850                                         DB_DEF_PDU_WRB_INDEX_SHIFT;
2851         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
2852
2853         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
2854 }
2855
2856 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
2857                               int *index, int *age)
2858 {
2859         *index = be32_to_cpu(itt) >> 16;
2860         if (age)
2861                 *age = conn->session->age;
2862 }
2863
2864 /**
2865  * beiscsi_alloc_pdu - allocates pdu and related resources
2866  * @task: libiscsi task
2867  * @opcode: opcode of pdu for task
2868  *
2869  * This is called with the session lock held. It will allocate
2870  * the wrb and sgl if needed for the command. And it will prep
2871  * the pdu's itt. beiscsi_parse_pdu will later translate
2872  * the pdu itt to the libiscsi task itt.
2873  */
2874 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2875 {
2876         struct beiscsi_io_task *io_task = task->dd_data;
2877         struct iscsi_conn *conn = task->conn;
2878         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2879         struct beiscsi_hba *phba = beiscsi_conn->phba;
2880         struct hwi_wrb_context *pwrb_context;
2881         struct hwi_controller *phwi_ctrlr;
2882         itt_t itt;
2883
2884         io_task->pwrb_handle = alloc_wrb_handle(phba,
2885                                                 beiscsi_conn->beiscsi_conn_cid,
2886                                                 task->itt);
2887         io_task->pwrb_handle->pio_handle = task;
2888         io_task->conn = beiscsi_conn;
2889
2890         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
2891         task->hdr_max = sizeof(struct be_cmd_bhs);
2892
2893         if (task->sc) {
2894                 spin_lock(&phba->io_sgl_lock);
2895                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
2896                 spin_unlock(&phba->io_sgl_lock);
2897                 if (!io_task->psgl_handle) {
2898                         phwi_ctrlr = phba->phwi_ctrlr;
2899                         pwrb_context = &phwi_ctrlr->wrb_context
2900                                         [beiscsi_conn->beiscsi_conn_cid];
2901                         free_wrb_handle(phba, pwrb_context,
2902                                                 io_task->pwrb_handle);
2903                         io_task->pwrb_handle = NULL;
2904                         SE_DEBUG(DBG_LVL_1,
2905                                  "Alloc of SGL_ICD Failed \n");
2906                         return -ENOMEM;
2907                 }
2908         } else {
2909                 io_task->scsi_cmnd = NULL;
2910                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
2911                         if (!beiscsi_conn->login_in_progress) {
2912                                 spin_lock(&phba->mgmt_sgl_lock);
2913                                 io_task->psgl_handle = (struct sgl_handle *)
2914                                                 alloc_mgmt_sgl_handle(phba);
2915                                 spin_unlock(&phba->mgmt_sgl_lock);
2916                                 if (!io_task->psgl_handle) {
2917                                         phwi_ctrlr = phba->phwi_ctrlr;
2918                                         pwrb_context =
2919                                         &phwi_ctrlr->wrb_context
2920                                         [beiscsi_conn->beiscsi_conn_cid];
2921                                         free_wrb_handle(phba, pwrb_context,
2922                                                         io_task->pwrb_handle);
2923                                         io_task->pwrb_handle = NULL;
2924                                         SE_DEBUG(DBG_LVL_1, "Alloc of "
2925                                                 "MGMT_SGL_ICD Failed \n");
2926                                         return -ENOMEM;
2927                                 }
2928                                 beiscsi_conn->login_in_progress = 1;
2929                                 beiscsi_conn->plogin_sgl_handle =
2930                                                         io_task->psgl_handle;
2931                         } else {
2932                                 io_task->psgl_handle =
2933                                                 beiscsi_conn->plogin_sgl_handle;
2934                         }
2935                 } else {
2936                         spin_lock(&phba->mgmt_sgl_lock);
2937                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
2938                         spin_unlock(&phba->mgmt_sgl_lock);
2939                         if (!io_task->psgl_handle) {
2940                                 phwi_ctrlr = phba->phwi_ctrlr;
2941                                 pwrb_context = &phwi_ctrlr->wrb_context
2942                                         [beiscsi_conn->beiscsi_conn_cid];
2943                                 free_wrb_handle(phba, pwrb_context,
2944                                                         io_task->pwrb_handle);
2945                                 io_task->pwrb_handle = NULL;
2946                                 SE_DEBUG(DBG_LVL_1, "Alloc of "
2947                                          "MGMT_SGL_ICD Failed \n");
2948                                 return -ENOMEM;
2949                         }
2950                 }
2951         }
2952         itt = (itt_t) cpu_to_be32(((unsigned int)task->itt << 16) |
2953                         (unsigned int)(io_task->psgl_handle->sgl_index));
2954         io_task->cmd_bhs->iscsi_hdr.itt = itt;
2955         return 0;
2956 }
2957
2958 static void beiscsi_cleanup_task(struct iscsi_task *task)
2959 {
2960         struct beiscsi_io_task *io_task = task->dd_data;
2961         struct iscsi_conn *conn = task->conn;
2962         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2963         struct beiscsi_hba *phba = beiscsi_conn->phba;
2964         struct hwi_wrb_context *pwrb_context;
2965         struct hwi_controller *phwi_ctrlr;
2966
2967         phwi_ctrlr = phba->phwi_ctrlr;
2968         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
2969         if (io_task->pwrb_handle) {
2970                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2971                 io_task->pwrb_handle = NULL;
2972         }
2973
2974         if (task->sc) {
2975                 if (io_task->psgl_handle) {
2976                         spin_lock(&phba->io_sgl_lock);
2977                         free_io_sgl_handle(phba, io_task->psgl_handle);
2978                         spin_unlock(&phba->io_sgl_lock);
2979                         io_task->psgl_handle = NULL;
2980                 }
2981         } else {
2982                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
2983                         return;
2984                 if (io_task->psgl_handle) {
2985                         spin_lock(&phba->mgmt_sgl_lock);
2986                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
2987                         spin_unlock(&phba->mgmt_sgl_lock);
2988                         io_task->psgl_handle = NULL;
2989                 }
2990         }
2991 }
2992
2993 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
2994                           unsigned int num_sg, unsigned int xferlen,
2995                           unsigned int writedir)
2996 {
2997
2998         struct beiscsi_io_task *io_task = task->dd_data;
2999         struct iscsi_conn *conn = task->conn;
3000         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3001         struct beiscsi_hba *phba = beiscsi_conn->phba;
3002         struct iscsi_wrb *pwrb = NULL;
3003         unsigned int doorbell = 0;
3004
3005         pwrb = io_task->pwrb_handle->pwrb;
3006
3007         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3008         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3009
3010         if (writedir) {
3011                 SE_DEBUG(DBG_LVL_4, " WRITE Command \t");
3012                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3013                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3014                               &io_task->cmd_bhs->iscsi_data_pdu,
3015                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3016                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3017                               &io_task->cmd_bhs->iscsi_data_pdu,
3018                               ISCSI_OPCODE_SCSI_DATA_OUT);
3019                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3020                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3021                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3022                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3023
3024         } else {
3025                 SE_DEBUG(DBG_LVL_4, "READ Command \t");
3026                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3027                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3028         }
3029         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3030                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3031                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3032
3033         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3034                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3035                                   lun[0]));
3036         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3037         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3038                       io_task->pwrb_handle->wrb_index);
3039         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3040                       be32_to_cpu(task->cmdsn));
3041         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3042                       io_task->psgl_handle->sgl_index);
3043
3044         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3045
3046         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3047                       io_task->pwrb_handle->nxt_wrb_index);
3048         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3049
3050         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3051         doorbell |= (io_task->pwrb_handle->wrb_index &
3052                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3053         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3054
3055         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3056         return 0;
3057 }
3058
3059 static int beiscsi_mtask(struct iscsi_task *task)
3060 {
3061         struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3062         struct iscsi_conn *conn = task->conn;
3063         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3064         struct beiscsi_hba *phba = beiscsi_conn->phba;
3065         struct iscsi_wrb *pwrb = NULL;
3066         unsigned int doorbell = 0;
3067         struct iscsi_task *aborted_task;
3068
3069         pwrb = io_task->pwrb_handle->pwrb;
3070         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3071                       be32_to_cpu(task->cmdsn));
3072         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3073                       io_task->pwrb_handle->wrb_index);
3074         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3075                       io_task->psgl_handle->sgl_index);
3076
3077         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3078         case ISCSI_OP_LOGIN:
3079                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, TGT_DM_CMD);
3080                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3081                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3082                 hwi_write_buffer(pwrb, task);
3083                 break;
3084         case ISCSI_OP_NOOP_OUT:
3085                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3086                 hwi_write_buffer(pwrb, task);
3087                 break;
3088         case ISCSI_OP_TEXT:
3089                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3090                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3091                 hwi_write_buffer(pwrb, task);
3092                 break;
3093         case ISCSI_OP_SCSI_TMFUNC:
3094                 aborted_task = iscsi_itt_to_task(conn,
3095                                         ((struct iscsi_tm *)task->hdr)->rtt);
3096                  if (!aborted_task)
3097                         return 0;
3098                 aborted_io_task = aborted_task->dd_data;
3099                 if (!aborted_io_task->scsi_cmnd)
3100                         return 0;
3101
3102                 mgmt_invalidate_icds(phba,
3103                                      aborted_io_task->psgl_handle->sgl_index,
3104                                      beiscsi_conn->beiscsi_conn_cid);
3105                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_TMF_CMD);
3106                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3107                 hwi_write_buffer(pwrb, task);
3108                 break;
3109         case ISCSI_OP_LOGOUT:
3110                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3111                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3112                                 HWH_TYPE_LOGOUT);
3113                 hwi_write_buffer(pwrb, task);
3114                 break;
3115
3116         default:
3117                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3118                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3119                 return -EINVAL;
3120         }
3121
3122         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3123                       be32_to_cpu(task->data_count));
3124         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3125                       io_task->pwrb_handle->nxt_wrb_index);
3126         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3127
3128         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3129         doorbell |= (io_task->pwrb_handle->wrb_index &
3130                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3131         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3132         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3133         return 0;
3134 }
3135
3136 static int beiscsi_task_xmit(struct iscsi_task *task)
3137 {
3138         struct iscsi_conn *conn = task->conn;
3139         struct beiscsi_io_task *io_task = task->dd_data;
3140         struct scsi_cmnd *sc = task->sc;
3141         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3142         struct scatterlist *sg;
3143         int num_sg;
3144         unsigned int  writedir = 0, xferlen = 0;
3145
3146         SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3147                  "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3148                  task, conn, beiscsi_conn);
3149         if (!sc)
3150                 return beiscsi_mtask(task);
3151
3152         io_task->scsi_cmnd = sc;
3153         num_sg = scsi_dma_map(sc);
3154         if (num_sg < 0) {
3155                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3156                 return num_sg;
3157         }
3158         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3159                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3160         xferlen = scsi_bufflen(sc);
3161         sg = scsi_sglist(sc);
3162         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3163                 writedir = 1;
3164                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3165                          task->imm_count);
3166         } else
3167                 writedir = 0;
3168         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3169 }
3170
3171 static void beiscsi_remove(struct pci_dev *pcidev)
3172 {
3173         struct beiscsi_hba *phba = NULL;
3174
3175         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3176         if (!phba) {
3177                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3178                 return;
3179         }
3180
3181         hwi_disable_intr(phba);
3182         if (phba->pcidev->irq)
3183                 free_irq(phba->pcidev->irq, phba);
3184         destroy_workqueue(phba->wq);
3185         if (blk_iopoll_enabled)
3186                 blk_iopoll_disable(&phba->iopoll);
3187
3188         beiscsi_clean_port(phba);
3189         beiscsi_free_mem(phba);
3190         beiscsi_unmap_pci_function(phba);
3191         pci_free_consistent(phba->pcidev,
3192                             phba->ctrl.mbox_mem_alloced.size,
3193                             phba->ctrl.mbox_mem_alloced.va,
3194                             phba->ctrl.mbox_mem_alloced.dma);
3195         iscsi_host_remove(phba->shost);
3196         pci_dev_put(phba->pcidev);
3197         iscsi_host_free(phba->shost);
3198 }
3199
3200 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3201                                 const struct pci_device_id *id)
3202 {
3203         struct beiscsi_hba *phba = NULL;
3204         int ret;
3205
3206         ret = beiscsi_enable_pci(pcidev);
3207         if (ret < 0) {
3208                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3209                              "Failed to enable pci device \n");
3210                 return ret;
3211         }
3212
3213         phba = beiscsi_hba_alloc(pcidev);
3214         if (!phba) {
3215                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3216                         " Failed in beiscsi_hba_alloc \n");
3217                 goto disable_pci;
3218         }
3219
3220         pci_set_drvdata(pcidev, phba);
3221         ret = be_ctrl_init(phba, pcidev);
3222         if (ret) {
3223                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3224                                 "Failed in be_ctrl_init\n");
3225                 goto hba_free;
3226         }
3227
3228         spin_lock_init(&phba->io_sgl_lock);
3229         spin_lock_init(&phba->mgmt_sgl_lock);
3230         spin_lock_init(&phba->isr_lock);
3231         beiscsi_get_params(phba);
3232         ret = beiscsi_init_port(phba);
3233         if (ret < 0) {
3234                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3235                              "Failed in beiscsi_init_port\n");
3236                 goto free_port;
3237         }
3238
3239         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3240                  phba->shost->host_no);
3241         phba->wq = create_singlethread_workqueue(phba->wq_name);
3242         if (!phba->wq) {
3243                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3244                                 "Failed to allocate work queue\n");
3245                 goto free_twq;
3246         }
3247
3248         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3249
3250         if (blk_iopoll_enabled) {
3251                 blk_iopoll_init(&phba->iopoll, be_iopoll_budget, be_iopoll);
3252                 blk_iopoll_enable(&phba->iopoll);
3253         }
3254
3255         ret = beiscsi_init_irqs(phba);
3256         if (ret < 0) {
3257                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3258                              "Failed to beiscsi_init_irqs\n");
3259                 goto free_blkenbld;
3260         }
3261         ret = hwi_enable_intr(phba);
3262         if (ret < 0) {
3263                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3264                              "Failed to hwi_enable_intr\n");
3265                 goto free_ctrlr;
3266         }
3267
3268         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3269         return 0;
3270
3271 free_ctrlr:
3272         if (phba->pcidev->irq)
3273                 free_irq(phba->pcidev->irq, phba);
3274 free_blkenbld:
3275         destroy_workqueue(phba->wq);
3276         if (blk_iopoll_enabled)
3277                 blk_iopoll_disable(&phba->iopoll);
3278 free_twq:
3279         beiscsi_clean_port(phba);
3280         beiscsi_free_mem(phba);
3281 free_port:
3282         pci_free_consistent(phba->pcidev,
3283                             phba->ctrl.mbox_mem_alloced.size,
3284                             phba->ctrl.mbox_mem_alloced.va,
3285                            phba->ctrl.mbox_mem_alloced.dma);
3286         beiscsi_unmap_pci_function(phba);
3287 hba_free:
3288         iscsi_host_remove(phba->shost);
3289         pci_dev_put(phba->pcidev);
3290         iscsi_host_free(phba->shost);
3291 disable_pci:
3292         pci_disable_device(pcidev);
3293         return ret;
3294 }
3295
3296 struct iscsi_transport beiscsi_iscsi_transport = {
3297         .owner = THIS_MODULE,
3298         .name = DRV_NAME,
3299         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3300                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3301         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3302                 ISCSI_MAX_XMIT_DLENGTH |
3303                 ISCSI_HDRDGST_EN |
3304                 ISCSI_DATADGST_EN |
3305                 ISCSI_INITIAL_R2T_EN |
3306                 ISCSI_MAX_R2T |
3307                 ISCSI_IMM_DATA_EN |
3308                 ISCSI_FIRST_BURST |
3309                 ISCSI_MAX_BURST |
3310                 ISCSI_PDU_INORDER_EN |
3311                 ISCSI_DATASEQ_INORDER_EN |
3312                 ISCSI_ERL |
3313                 ISCSI_CONN_PORT |
3314                 ISCSI_CONN_ADDRESS |
3315                 ISCSI_EXP_STATSN |
3316                 ISCSI_PERSISTENT_PORT |
3317                 ISCSI_PERSISTENT_ADDRESS |
3318                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3319                 ISCSI_USERNAME | ISCSI_PASSWORD |
3320                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3321                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3322                 ISCSI_LU_RESET_TMO |
3323                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3324                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3325         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3326                                 ISCSI_HOST_INITIATOR_NAME,
3327         .create_session = beiscsi_session_create,
3328         .destroy_session = beiscsi_session_destroy,
3329         .create_conn = beiscsi_conn_create,
3330         .bind_conn = beiscsi_conn_bind,
3331         .destroy_conn = iscsi_conn_teardown,
3332         .set_param = beiscsi_set_param,
3333         .get_conn_param = beiscsi_conn_get_param,
3334         .get_session_param = iscsi_session_get_param,
3335         .get_host_param = beiscsi_get_host_param,
3336         .start_conn = beiscsi_conn_start,
3337         .stop_conn = beiscsi_conn_stop,
3338         .send_pdu = iscsi_conn_send_pdu,
3339         .xmit_task = beiscsi_task_xmit,
3340         .cleanup_task = beiscsi_cleanup_task,
3341         .alloc_pdu = beiscsi_alloc_pdu,
3342         .parse_pdu_itt = beiscsi_parse_pdu,
3343         .get_stats = beiscsi_conn_get_stats,
3344         .ep_connect = beiscsi_ep_connect,
3345         .ep_poll = beiscsi_ep_poll,
3346         .ep_disconnect = beiscsi_ep_disconnect,
3347         .session_recovery_timedout = iscsi_session_recovery_timedout,
3348 };
3349
3350 static struct pci_driver beiscsi_pci_driver = {
3351         .name = DRV_NAME,
3352         .probe = beiscsi_dev_probe,
3353         .remove = beiscsi_remove,
3354         .id_table = beiscsi_pci_id_table
3355 };
3356
3357 static int __init beiscsi_module_init(void)
3358 {
3359         int ret;
3360
3361         beiscsi_scsi_transport =
3362                         iscsi_register_transport(&beiscsi_iscsi_transport);
3363         if (!beiscsi_scsi_transport) {
3364                 SE_DEBUG(DBG_LVL_1,
3365                          "beiscsi_module_init - Unable to  register beiscsi"
3366                          "transport.\n");
3367                 ret = -ENOMEM;
3368         }
3369         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3370                  &beiscsi_iscsi_transport);
3371
3372         ret = pci_register_driver(&beiscsi_pci_driver);
3373         if (ret) {
3374                 SE_DEBUG(DBG_LVL_1,
3375                          "beiscsi_module_init - Unable to  register"
3376                          "beiscsi pci driver.\n");
3377                 goto unregister_iscsi_transport;
3378         }
3379         return 0;
3380
3381 unregister_iscsi_transport:
3382         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3383         return ret;
3384 }
3385
3386 static void __exit beiscsi_module_exit(void)
3387 {
3388         pci_unregister_driver(&beiscsi_pci_driver);
3389         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3390 }
3391
3392 module_init(beiscsi_module_init);
3393 module_exit(beiscsi_module_exit);