drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2009 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@serverengines.com
  14  *
  15  *  ServerEngines
  16  * 209 N. Fair Oaks Ave
  17  * Sunnyvale, CA 94085
  18  *
  19  */
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/blkdev.h>
  24 #include <linux/pci.h>
  25 #include <linux/string.h>
  26 #include <linux/kernel.h>
  27 #include <linux/semaphore.h>
  28
  29 #include <scsi/libiscsi.h>
  30 #include <scsi/scsi_transport_iscsi.h>
  31 #include <scsi/scsi_transport.h>
  32 #include <scsi/scsi_cmnd.h>
  33 #include <scsi/scsi_device.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi.h>
  36 #include "be_main.h"
  37 #include "be_iscsi.h"
  38 #include "be_mgmt.h"
  39
  40 static unsigned int be_iopoll_budget = 10;
  41 static unsigned int be_max_phys_size = 64;
  42 static unsigned int enable_msix;
  43
  44 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  45 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  46 MODULE_AUTHOR("ServerEngines Corporation");
  47 MODULE_LICENSE("GPL");
  48 module_param(be_iopoll_budget, int, 0);
  49 module_param(enable_msix, int, 0);
  50 module_param(be_max_phys_size, uint, S_IRUGO);
  51 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  52                                    "contiguous memory that can be allocated."
  53                                    "Range is 16 - 128");
  54
  55 static int beiscsi_slave_configure(struct scsi_device *sdev)
  56 {
  57         blk_queue_max_segment_size(sdev->request_queue, 65536);
  58         return 0;
  59 }
  60
  61 static struct scsi_host_template beiscsi_sht = {
  62         .module = THIS_MODULE,
  63         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
  64         .proc_name = DRV_NAME,
  65         .queuecommand = iscsi_queuecommand,
  66         .eh_abort_handler = iscsi_eh_abort,
  67         .change_queue_depth = iscsi_change_queue_depth,
  68         .slave_configure = beiscsi_slave_configure,
  69         .target_alloc = iscsi_target_alloc,
  70         .eh_device_reset_handler = iscsi_eh_device_reset,
  71         .eh_target_reset_handler = iscsi_eh_target_reset,
  72         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
  73         .can_queue = BE2_IO_DEPTH,
  74         .this_id = -1,
  75         .max_sectors = BEISCSI_MAX_SECTORS,
  76         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
  77         .use_clustering = ENABLE_CLUSTERING,
  78 };
  79 static struct scsi_transport_template *beiscsi_scsi_transport;
  80
  81 /*------------------- PCI Driver operations and data ----------------- */
  82 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
  83         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  84         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  85         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  86         { 0 }
  87 };
  88 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  89
  90 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
  91 {
  92         struct beiscsi_hba *phba;
  93         struct Scsi_Host *shost;
  94
  95         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
  96         if (!shost) {
  97                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
  98                         "iscsi_host_alloc failed \n");
  99                 return NULL;
 100         }
 101         shost->dma_boundary = pcidev->dma_mask;
 102         shost->max_id = BE2_MAX_SESSIONS;
 103         shost->max_channel = 0;
 104         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 105         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 106         shost->transportt = beiscsi_scsi_transport;
 107
 108         phba = iscsi_host_priv(shost);
 109         memset(phba, 0, sizeof(*phba));
 110         phba->shost = shost;
 111         phba->pcidev = pci_dev_get(pcidev);
 112
 113         if (iscsi_host_add(shost, &phba->pcidev->dev))
 114                 goto free_devices;
 115         return phba;
 116
 117 free_devices:
 118         pci_dev_put(phba->pcidev);
 119         iscsi_host_free(phba->shost);
 120         return NULL;
 121 }
 122
 123 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 124 {
 125         if (phba->csr_va) {
 126                 iounmap(phba->csr_va);
 127                 phba->csr_va = NULL;
 128         }
 129         if (phba->db_va) {
 130                 iounmap(phba->db_va);
 131                 phba->db_va = NULL;
 132         }
 133         if (phba->pci_va) {
 134                 iounmap(phba->pci_va);
 135                 phba->pci_va = NULL;
 136         }
 137 }
 138
 139 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 140                                 struct pci_dev *pcidev)
 141 {
 142         u8 __iomem *addr;
 143
 144         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 145                                pci_resource_len(pcidev, 2));
 146         if (addr == NULL)
 147                 return -ENOMEM;
 148         phba->ctrl.csr = addr;
 149         phba->csr_va = addr;
 150         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 151
 152         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 153         if (addr == NULL)
 154                 goto pci_map_err;
 155         phba->ctrl.db = addr;
 156         phba->db_va = addr;
 157         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 158
 159         addr = ioremap_nocache(pci_resource_start(pcidev, 1),
 160                                pci_resource_len(pcidev, 1));
 161         if (addr == NULL)
 162                 goto pci_map_err;
 163         phba->ctrl.pcicfg = addr;
 164         phba->pci_va = addr;
 165         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
 166         return 0;
 167
 168 pci_map_err:
 169         beiscsi_unmap_pci_function(phba);
 170         return -ENOMEM;
 171 }
 172
 173 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 174 {
 175         int ret;
 176
 177         ret = pci_enable_device(pcidev);
 178         if (ret) {
 179                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 180                         "failed. Returning -ENODEV\n");
 181                 return ret;
 182         }
 183
 184         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 185                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 186                 if (ret) {
 187                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 188                         pci_disable_device(pcidev);
 189                         return ret;
 190                 }
 191         }
 192         return 0;
 193 }
 194
 195 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 196 {
 197         struct be_ctrl_info *ctrl = &phba->ctrl;
 198         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 199         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 200         int status = 0;
 201
 202         ctrl->pdev = pdev;
 203         status = beiscsi_map_pci_bars(phba, pdev);
 204         if (status)
 205                 return status;
 206
 207         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 208         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 209                                                   mbox_mem_alloc->size,
 210                                                   &mbox_mem_alloc->dma);
 211         if (!mbox_mem_alloc->va) {
 212                 beiscsi_unmap_pci_function(phba);
 213                 status = -ENOMEM;
 214                 return status;
 215         }
 216
 217         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 218         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 219         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 220         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 221         spin_lock_init(&ctrl->mbox_lock);
 222         return status;
 223 }
 224
 225 static void beiscsi_get_params(struct beiscsi_hba *phba)
 226 {
 227         phba->params.ios_per_ctrl = BE2_IO_DEPTH;
 228         phba->params.cxns_per_ctrl = BE2_MAX_SESSIONS;
 229         phba->params.asyncpdus_per_ctrl = BE2_ASYNCPDUS;
 230         phba->params.icds_per_ctrl = BE2_MAX_ICDS / 2;
 231         phba->params.num_sge_per_io = BE2_SGE;
 232         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 233         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 234         phba->params.eq_timer = 64;
 235         phba->params.num_eq_entries =
 236             (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
 237                                                                 512) + 1) * 512;
 238         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 239                                 ? 1024 : phba->params.num_eq_entries;
 240         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
 241                  phba->params.num_eq_entries);
 242         phba->params.num_cq_entries =
 243             (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
 244                                                                 512) + 1) * 512;
 245         SE_DEBUG(DBG_LVL_8,
 246                 "phba->params.num_cq_entries=%d BE2_CMDS_PER_CXN=%d"
 247                 "BE2_LOGOUTS=%d BE2_TMFS=%d BE2_ASYNCPDUS=%d \n",
 248                 phba->params.num_cq_entries, BE2_CMDS_PER_CXN,
 249                 BE2_LOGOUTS, BE2_TMFS, BE2_ASYNCPDUS);
 250         phba->params.wrbs_per_cxn = 256;
 251 }
 252
 253 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 254                            unsigned int id, unsigned int clr_interrupt,
 255                            unsigned int num_processed,
 256                            unsigned char rearm, unsigned char event)
 257 {
 258         u32 val = 0;
 259         val |= id & DB_EQ_RING_ID_MASK;
 260         if (rearm)
 261                 val |= 1 << DB_EQ_REARM_SHIFT;
 262         if (clr_interrupt)
 263                 val |= 1 << DB_EQ_CLR_SHIFT;
 264         if (event)
 265                 val |= 1 << DB_EQ_EVNT_SHIFT;
 266         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 267         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 268 }
 269
 270 /**
 271  * be_isr - The isr routine of the driver.
 272  * @irq: Not used
 273  * @dev_id: Pointer to host adapter structure
 274  */
 275 static irqreturn_t be_isr(int irq, void *dev_id)
 276 {
 277         struct beiscsi_hba *phba;
 278         struct hwi_controller *phwi_ctrlr;
 279         struct hwi_context_memory *phwi_context;
 280         struct be_eq_entry *eqe = NULL;
 281         struct be_queue_info *eq;
 282         struct be_queue_info *cq;
 283         unsigned long flags, index;
 284         unsigned int num_eq_processed;
 285         struct be_ctrl_info *ctrl;
 286         int isr;
 287
 288         phba = dev_id;
 289         if (!enable_msix) {
 290                 ctrl = &phba->ctrl;;
 291                 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 292                                (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 293                 if (!isr)
 294                         return IRQ_NONE;
 295         }
 296
 297         phwi_ctrlr = phba->phwi_ctrlr;
 298         phwi_context = phwi_ctrlr->phwi_ctxt;
 299         eq = &phwi_context->be_eq.q;
 300         cq = &phwi_context->be_cq;
 301         index = 0;
 302         eqe = queue_tail_node(eq);
 303         if (!eqe)
 304                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 305
 306         num_eq_processed = 0;
 307         if (blk_iopoll_enabled) {
 308                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 309                                         & EQE_VALID_MASK) {
 310                         if (!blk_iopoll_sched_prep(&phba->iopoll))
 311                                 blk_iopoll_sched(&phba->iopoll);
 312
 313                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 314                         queue_tail_inc(eq);
 315                         eqe = queue_tail_node(eq);
 316                         num_eq_processed++;
 317                         SE_DEBUG(DBG_LVL_8, "Valid EQE\n");
 318                 }
 319                 if (num_eq_processed) {
 320                         hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 0, 1);
 321                         return IRQ_HANDLED;
 322                 } else
 323                         return IRQ_NONE;
 324         } else {
 325                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 326                                                 & EQE_VALID_MASK) {
 327
 328                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 329                              resource_id) / 32] &
 330                              EQE_RESID_MASK) >> 16) != cq->id) {
 331                                 spin_lock_irqsave(&phba->isr_lock, flags);
 332                                 phba->todo_mcc_cq = 1;
 333                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 334                         } else {
 335                                 spin_lock_irqsave(&phba->isr_lock, flags);
 336                                 phba->todo_cq = 1;
 337                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 338                         }
 339                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 340                         queue_tail_inc(eq);
 341                         eqe = queue_tail_node(eq);
 342                         num_eq_processed++;
 343                 }
 344                 if (phba->todo_cq || phba->todo_mcc_cq)
 345                         queue_work(phba->wq, &phba->work_cqs);
 346
 347                 if (num_eq_processed) {
 348                         hwi_ring_eq_db(phba, eq->id, 0, num_eq_processed, 1, 1);
 349                         return IRQ_HANDLED;
 350                 } else
 351                         return IRQ_NONE;
 352         }
 353 }
 354
 355 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 356 {
 357         struct pci_dev *pcidev = phba->pcidev;
 358         int ret;
 359
 360         ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, "beiscsi", phba);
 361         if (ret) {
 362                 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 363                              "Failed to register irq\\n");
 364                 return ret;
 365         }
 366         return 0;
 367 }
 368
 369 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 370                            unsigned int id, unsigned int num_processed,
 371                            unsigned char rearm, unsigned char event)
 372 {
 373         u32 val = 0;
 374         val |= id & DB_CQ_RING_ID_MASK;
 375         if (rearm)
 376                 val |= 1 << DB_CQ_REARM_SHIFT;
 377         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 378         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 379 }
 380
 381 /*
 382  * async pdus include
 383  * a. unsolicited NOP-In (target initiated NOP-In)
 384  * b. Async Messages
 385  * c. Reject PDU
 386  * d. Login response
 387  * These headers arrive unprocessed by the EP firmware and iSCSI layer
 388  * process them
 389  */
 390 static unsigned int
 391 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 392                           struct beiscsi_hba *phba,
 393                           unsigned short cid,
 394                           struct pdu_base *ppdu,
 395                           unsigned long pdu_len,
 396                           void *pbuffer, unsigned long buf_len)
 397 {
 398         struct iscsi_conn *conn = beiscsi_conn->conn;
 399         struct iscsi_session *session = conn->session;
 400
 401         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 402                                                 PDUBASE_OPCODE_MASK) {
 403         case ISCSI_OP_NOOP_IN:
 404                 pbuffer = NULL;
 405                 buf_len = 0;
 406                 break;
 407         case ISCSI_OP_ASYNC_EVENT:
 408                 break;
 409         case ISCSI_OP_REJECT:
 410                 WARN_ON(!pbuffer);
 411                 WARN_ON(!(buf_len == 48));
 412                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 413                 break;
 414         case ISCSI_OP_LOGIN_RSP:
 415                 break;
 416         default:
 417                 shost_printk(KERN_WARNING, phba->shost,
 418                              "Unrecognized opcode 0x%x in async msg \n",
 419                              (ppdu->
 420                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 421                                                 & PDUBASE_OPCODE_MASK));
 422                 return 1;
 423         }
 424
 425         spin_lock_bh(&session->lock);
 426         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 427         spin_unlock_bh(&session->lock);
 428         return 0;
 429 }
 430
 431 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 432 {
 433         struct sgl_handle *psgl_handle;
 434
 435         if (phba->io_sgl_hndl_avbl) {
 436                 SE_DEBUG(DBG_LVL_8,
 437                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
 438                          phba->io_sgl_alloc_index);
 439                 psgl_handle = phba->io_sgl_hndl_base[phba->
 440                                                 io_sgl_alloc_index];
 441                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 442                 phba->io_sgl_hndl_avbl--;
 443                 if (phba->io_sgl_alloc_index == (phba->params.ios_per_ctrl - 1))
 444                         phba->io_sgl_alloc_index = 0;
 445                 else
 446                         phba->io_sgl_alloc_index++;
 447         } else
 448                 psgl_handle = NULL;
 449         return psgl_handle;
 450 }
 451
 452 static void
 453 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 454 {
 455         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
 456                  phba->io_sgl_free_index);
 457         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 458                 /*
 459                  * this can happen if clean_task is called on a task that
 460                  * failed in xmit_task or alloc_pdu.
 461                  */
 462                  SE_DEBUG(DBG_LVL_8,
 463                          "Double Free in IO SGL io_sgl_free_index=%d,"
 464                          "value there=%p \n", phba->io_sgl_free_index,
 465                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 466                 return;
 467         }
 468         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 469         phba->io_sgl_hndl_avbl++;
 470         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 471                 phba->io_sgl_free_index = 0;
 472         else
 473                 phba->io_sgl_free_index++;
 474 }
 475
 476 /**
 477  * alloc_wrb_handle - To allocate a wrb handle
 478  * @phba: The hba pointer
 479  * @cid: The cid to use for allocation
 480  * @index: index allocation and wrb index
 481  *
 482  * This happens under session_lock until submission to chip
 483  */
 484 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
 485                                     int index)
 486 {
 487         struct hwi_wrb_context *pwrb_context;
 488         struct hwi_controller *phwi_ctrlr;
 489         struct wrb_handle *pwrb_handle;
 490
 491         phwi_ctrlr = phba->phwi_ctrlr;
 492         pwrb_context = &phwi_ctrlr->wrb_context[cid];
 493         pwrb_handle = pwrb_context->pwrb_handle_base[index];
 494         pwrb_handle->wrb_index = index;
 495         pwrb_handle->nxt_wrb_index = index;
 496         return pwrb_handle;
 497 }
 498
 499 /**
 500  * free_wrb_handle - To free the wrb handle back to pool
 501  * @phba: The hba pointer
 502  * @pwrb_context: The context to free from
 503  * @pwrb_handle: The wrb_handle to free
 504  *
 505  * This happens under session_lock until submission to chip
 506  */
 507 static void
 508 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
 509                 struct wrb_handle *pwrb_handle)
 510 {
 511         SE_DEBUG(DBG_LVL_8,
 512                  "FREE WRB: pwrb_handle=%p free_index=%d=0x%x"
 513                  "wrb_handles_available=%d \n",
 514                  pwrb_handle, pwrb_context->free_index,
 515                  pwrb_context->free_index, pwrb_context->wrb_handles_available);
 516 }
 517
 518 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
 519 {
 520         struct sgl_handle *psgl_handle;
 521
 522         if (phba->eh_sgl_hndl_avbl) {
 523                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
 524                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
 525                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
 526                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
 527                 phba->eh_sgl_hndl_avbl--;
 528                 if (phba->eh_sgl_alloc_index ==
 529                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
 530                      1))
 531                         phba->eh_sgl_alloc_index = 0;
 532                 else
 533                         phba->eh_sgl_alloc_index++;
 534         } else
 535                 psgl_handle = NULL;
 536         return psgl_handle;
 537 }
 538
 539 void
 540 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 541 {
 542
 543         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
 544                 /*
 545                  * this can happen if clean_task is called on a task that
 546                  * failed in xmit_task or alloc_pdu.
 547                  */
 548                 SE_DEBUG(DBG_LVL_8,
 549                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
 550                          phba->eh_sgl_free_index);
 551                 return;
 552         }
 553         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
 554         phba->eh_sgl_hndl_avbl++;
 555         if (phba->eh_sgl_free_index ==
 556             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
 557                 phba->eh_sgl_free_index = 0;
 558         else
 559                 phba->eh_sgl_free_index++;
 560 }
 561
 562 static void
 563 be_complete_io(struct beiscsi_conn *beiscsi_conn,
 564                struct iscsi_task *task, struct sol_cqe *psol)
 565 {
 566         struct beiscsi_io_task *io_task = task->dd_data;
 567         struct be_status_bhs *sts_bhs =
 568                                 (struct be_status_bhs *)io_task->cmd_bhs;
 569         struct iscsi_conn *conn = beiscsi_conn->conn;
 570         unsigned int sense_len;
 571         unsigned char *sense;
 572         u32 resid = 0, exp_cmdsn, max_cmdsn;
 573         u8 rsp, status, flags;
 574
 575         exp_cmdsn = be32_to_cpu(psol->
 576                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 577                         & SOL_EXP_CMD_SN_MASK);
 578         max_cmdsn = be32_to_cpu((psol->
 579                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 580                         & SOL_EXP_CMD_SN_MASK) +
 581                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 582                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 583         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
 584                                                 & SOL_RESP_MASK) >> 16);
 585         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
 586                                                 & SOL_STS_MASK) >> 8);
 587         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 588                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 589
 590         task->sc->result = (DID_OK << 16) | status;
 591         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
 592                 task->sc->result = DID_ERROR << 16;
 593                 goto unmap;
 594         }
 595
 596         /* bidi not initially supported */
 597         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
 598                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
 599                                 32] & SOL_RES_CNT_MASK);
 600
 601                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
 602                         task->sc->result = DID_ERROR << 16;
 603
 604                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 605                         scsi_set_resid(task->sc, resid);
 606                         if (!status && (scsi_bufflen(task->sc) - resid <
 607                             task->sc->underflow))
 608                                 task->sc->result = DID_ERROR << 16;
 609                 }
 610         }
 611
 612         if (status == SAM_STAT_CHECK_CONDITION) {
 613                 sense = sts_bhs->sense_info + sizeof(unsigned short);
 614                 sense_len =
 615                     cpu_to_be16((unsigned short)(sts_bhs->sense_info[0]));
 616                 memcpy(task->sc->sense_buffer, sense,
 617                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
 618         }
 619         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
 620                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 621                                                         & SOL_RES_CNT_MASK)
 622                          conn->rxdata_octets += (psol->
 623                               dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 624                                                         & SOL_RES_CNT_MASK);
 625         }
 626 unmap:
 627         scsi_dma_unmap(io_task->scsi_cmnd);
 628         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
 629 }
 630
 631 static void
 632 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
 633                    struct iscsi_task *task, struct sol_cqe *psol)
 634 {
 635         struct iscsi_logout_rsp *hdr;
 636         struct iscsi_conn *conn = beiscsi_conn->conn;
 637
 638         hdr = (struct iscsi_logout_rsp *)task->hdr;
 639         hdr->t2wait = 5;
 640         hdr->t2retain = 0;
 641         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 642                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 643         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 644                                         32] & SOL_RESP_MASK);
 645         hdr->exp_cmdsn = cpu_to_be32(psol->
 646                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 647                                         & SOL_EXP_CMD_SN_MASK);
 648         hdr->max_cmdsn = be32_to_cpu((psol->
 649                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 650                                         & SOL_EXP_CMD_SN_MASK) +
 651                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 652                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 653         hdr->hlength = 0;
 654
 655         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 656 }
 657
 658 static void
 659 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
 660                 struct iscsi_task *task, struct sol_cqe *psol)
 661 {
 662         struct iscsi_tm_rsp *hdr;
 663         struct iscsi_conn *conn = beiscsi_conn->conn;
 664
 665         hdr = (struct iscsi_tm_rsp *)task->hdr;
 666         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 667                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 668         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 669                                         32] & SOL_RESP_MASK);
 670         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 671                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 672         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 673                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 674                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 675                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 676         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 677 }
 678
 679 static void
 680 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 681                        struct beiscsi_hba *phba, struct sol_cqe *psol)
 682 {
 683         struct hwi_wrb_context *pwrb_context;
 684         struct wrb_handle *pwrb_handle;
 685         struct hwi_controller *phwi_ctrlr;
 686         struct iscsi_conn *conn = beiscsi_conn->conn;
 687         struct iscsi_session *session = conn->session;
 688
 689         phwi_ctrlr = phba->phwi_ctrlr;
 690         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
 691                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
 692                                 SOL_CID_MASK) >> 6)];
 693         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
 694                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
 695                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
 696         spin_lock_bh(&session->lock);
 697         free_wrb_handle(phba, pwrb_context, pwrb_handle);
 698         spin_unlock_bh(&session->lock);
 699 }
 700
 701 static void
 702 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
 703                        struct iscsi_task *task, struct sol_cqe *psol)
 704 {
 705         struct iscsi_nopin *hdr;
 706         struct iscsi_conn *conn = beiscsi_conn->conn;
 707
 708         hdr = (struct iscsi_nopin *)task->hdr;
 709         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 710                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 711         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 712                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 713         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 714                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 715                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 716                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 717         hdr->opcode = ISCSI_OP_NOOP_IN;
 718         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 719 }
 720
 721 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
 722                              struct beiscsi_hba *phba, struct sol_cqe *psol)
 723 {
 724         struct hwi_wrb_context *pwrb_context;
 725         struct wrb_handle *pwrb_handle;
 726         struct iscsi_wrb *pwrb = NULL;
 727         struct hwi_controller *phwi_ctrlr;
 728         struct iscsi_task *task;
 729         struct beiscsi_io_task *io_task;
 730         struct iscsi_conn *conn = beiscsi_conn->conn;
 731         struct iscsi_session *session = conn->session;
 732
 733         phwi_ctrlr = phba->phwi_ctrlr;
 734
 735         pwrb_context = &phwi_ctrlr->
 736                 wrb_context[((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
 737                 & SOL_CID_MASK) >> 6)];
 738         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
 739                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
 740                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
 741
 742         task = pwrb_handle->pio_handle;
 743         io_task = task->dd_data;
 744         spin_lock_bh(&session->lock);
 745         pwrb = pwrb_handle->pwrb;
 746         switch ((pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
 747                  WRB_TYPE_MASK) >> 28) {
 748         case HWH_TYPE_IO:
 749         case HWH_TYPE_IO_RD:
 750                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
 751                     ISCSI_OP_NOOP_OUT) {
 752                         be_complete_nopin_resp(beiscsi_conn, task, psol);
 753                 } else
 754                         be_complete_io(beiscsi_conn, task, psol);
 755                 break;
 756
 757         case HWH_TYPE_LOGOUT:
 758                 be_complete_logout(beiscsi_conn, task, psol);
 759                 break;
 760
 761         case HWH_TYPE_LOGIN:
 762                 SE_DEBUG(DBG_LVL_1,
 763                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
 764                          "- Solicited path \n");
 765                 break;
 766
 767         case HWH_TYPE_TMF:
 768                 be_complete_tmf(beiscsi_conn, task, psol);
 769                 break;
 770
 771         case HWH_TYPE_NOP:
 772                 be_complete_nopin_resp(beiscsi_conn, task, psol);
 773                 break;
 774
 775         default:
 776                 shost_printk(KERN_WARNING, phba->shost,
 777                             "wrb_index 0x%x CID 0x%x\n",
 778                             ((psol->dw[offsetof(struct amap_iscsi_wrb, type) /
 779                                         32] & SOL_WRB_INDEX_MASK) >> 16),
 780                             ((psol->dw[offsetof(struct amap_sol_cqe, cid) / 32]
 781                                         & SOL_CID_MASK) >> 6));
 782                 break;
 783         }
 784
 785         spin_unlock_bh(&session->lock);
 786 }
 787
 788 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
 789                                           *pasync_ctx, unsigned int is_header,
 790                                           unsigned int host_write_ptr)
 791 {
 792         if (is_header)
 793                 return &pasync_ctx->async_entry[host_write_ptr].
 794                     header_busy_list;
 795         else
 796                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
 797 }
 798
 799 static struct async_pdu_handle *
 800 hwi_get_async_handle(struct beiscsi_hba *phba,
 801                      struct beiscsi_conn *beiscsi_conn,
 802                      struct hwi_async_pdu_context *pasync_ctx,
 803                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
 804 {
 805         struct be_bus_address phys_addr;
 806         struct list_head *pbusy_list;
 807         struct async_pdu_handle *pasync_handle = NULL;
 808         int buffer_len = 0;
 809         unsigned char buffer_index = -1;
 810         unsigned char is_header = 0;
 811
 812         phys_addr.u.a32.address_lo =
 813             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
 814             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
 815                                                 & PDUCQE_DPL_MASK) >> 16);
 816         phys_addr.u.a32.address_hi =
 817             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
 818
 819         phys_addr.u.a64.address =
 820                         *((unsigned long long *)(&phys_addr.u.a64.address));
 821
 822         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
 823                         & PDUCQE_CODE_MASK) {
 824         case UNSOL_HDR_NOTIFY:
 825                 is_header = 1;
 826
 827                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
 828                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
 829                         index) / 32] & PDUCQE_INDEX_MASK));
 830
 831                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
 832                                 pasync_ctx->async_header.pa_base.u.a64.address);
 833
 834                 buffer_index = buffer_len /
 835                                 pasync_ctx->async_header.buffer_size;
 836
 837                 break;
 838         case UNSOL_DATA_NOTIFY:
 839                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
 840                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
 841                                         index) / 32] & PDUCQE_INDEX_MASK));
 842                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
 843                                         pasync_ctx->async_data.pa_base.u.
 844                                         a64.address);
 845                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
 846                 break;
 847         default:
 848                 pbusy_list = NULL;
 849                 shost_printk(KERN_WARNING, phba->shost,
 850                         "Unexpected code=%d \n",
 851                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
 852                                         code) / 32] & PDUCQE_CODE_MASK);
 853                 return NULL;
 854         }
 855
 856         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
 857         WARN_ON(list_empty(pbusy_list));
 858         list_for_each_entry(pasync_handle, pbusy_list, link) {
 859                 WARN_ON(pasync_handle->consumed);
 860                 if (pasync_handle->index == buffer_index)
 861                         break;
 862         }
 863
 864         WARN_ON(!pasync_handle);
 865
 866         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid;
 867         pasync_handle->is_header = is_header;
 868         pasync_handle->buffer_len = ((pdpdu_cqe->
 869                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
 870                         & PDUCQE_DPL_MASK) >> 16);
 871
 872         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
 873                         index) / 32] & PDUCQE_INDEX_MASK);
 874         return pasync_handle;
 875 }
 876
 877 static unsigned int
 878 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
 879                            unsigned int is_header, unsigned int cq_index)
 880 {
 881         struct list_head *pbusy_list;
 882         struct async_pdu_handle *pasync_handle;
 883         unsigned int num_entries, writables = 0;
 884         unsigned int *pep_read_ptr, *pwritables;
 885
 886
 887         if (is_header) {
 888                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
 889                 pwritables = &pasync_ctx->async_header.writables;
 890                 num_entries = pasync_ctx->async_header.num_entries;
 891         } else {
 892                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
 893                 pwritables = &pasync_ctx->async_data.writables;
 894                 num_entries = pasync_ctx->async_data.num_entries;
 895         }
 896
 897         while ((*pep_read_ptr) != cq_index) {
 898                 (*pep_read_ptr)++;
 899                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
 900
 901                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
 902                                                      *pep_read_ptr);
 903                 if (writables == 0)
 904                         WARN_ON(list_empty(pbusy_list));
 905
 906                 if (!list_empty(pbusy_list)) {
 907                         pasync_handle = list_entry(pbusy_list->next,
 908                                                    struct async_pdu_handle,
 909                                                    link);
 910                         WARN_ON(!pasync_handle);
 911                         pasync_handle->consumed = 1;
 912                 }
 913
 914                 writables++;
 915         }
 916
 917         if (!writables) {
 918                 SE_DEBUG(DBG_LVL_1,
 919                          "Duplicate notification received - index 0x%x!!\n",
 920                          cq_index);
 921                 WARN_ON(1);
 922         }
 923
 924         *pwritables = *pwritables + writables;
 925         return 0;
 926 }
 927
 928 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
 929                                        unsigned int cri)
 930 {
 931         struct hwi_controller *phwi_ctrlr;
 932         struct hwi_async_pdu_context *pasync_ctx;
 933         struct async_pdu_handle *pasync_handle, *tmp_handle;
 934         struct list_head *plist;
 935         unsigned int i = 0;
 936
 937         phwi_ctrlr = phba->phwi_ctrlr;
 938         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
 939
 940         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
 941
 942         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
 943                 list_del(&pasync_handle->link);
 944
 945                 if (i == 0) {
 946                         list_add_tail(&pasync_handle->link,
 947                                       &pasync_ctx->async_header.free_list);
 948                         pasync_ctx->async_header.free_entries++;
 949                         i++;
 950                 } else {
 951                         list_add_tail(&pasync_handle->link,
 952                                       &pasync_ctx->async_data.free_list);
 953                         pasync_ctx->async_data.free_entries++;
 954                         i++;
 955                 }
 956         }
 957
 958         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
 959         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
 960         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
 961         return 0;
 962 }
 963
 964 static struct phys_addr *
 965 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
 966                      unsigned int is_header, unsigned int host_write_ptr)
 967 {
 968         struct phys_addr *pasync_sge = NULL;
 969
 970         if (is_header)
 971                 pasync_sge = pasync_ctx->async_header.ring_base;
 972         else
 973                 pasync_sge = pasync_ctx->async_data.ring_base;
 974
 975         return pasync_sge + host_write_ptr;
 976 }
 977
 978 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
 979                                    unsigned int is_header)
 980 {
 981         struct hwi_controller *phwi_ctrlr;
 982         struct hwi_async_pdu_context *pasync_ctx;
 983         struct async_pdu_handle *pasync_handle;
 984         struct list_head *pfree_link, *pbusy_list;
 985         struct phys_addr *pasync_sge;
 986         unsigned int ring_id, num_entries;
 987         unsigned int host_write_num;
 988         unsigned int writables;
 989         unsigned int i = 0;
 990         u32 doorbell = 0;
 991
 992         phwi_ctrlr = phba->phwi_ctrlr;
 993         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
 994
 995         if (is_header) {
 996                 num_entries = pasync_ctx->async_header.num_entries;
 997                 writables = min(pasync_ctx->async_header.writables,
 998                                 pasync_ctx->async_header.free_entries);
 999                 pfree_link = pasync_ctx->async_header.free_list.next;
1000                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1001                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1002         } else {
1003                 num_entries = pasync_ctx->async_data.num_entries;
1004                 writables = min(pasync_ctx->async_data.writables,
1005                                 pasync_ctx->async_data.free_entries);
1006                 pfree_link = pasync_ctx->async_data.free_list.next;
1007                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1008                 ring_id = phwi_ctrlr->default_pdu_data.id;
1009         }
1010
1011         writables = (writables / 8) * 8;
1012         if (writables) {
1013                 for (i = 0; i < writables; i++) {
1014                         pbusy_list =
1015                             hwi_get_async_busy_list(pasync_ctx, is_header,
1016                                                     host_write_num);
1017                         pasync_handle =
1018                             list_entry(pfree_link, struct async_pdu_handle,
1019                                                                 link);
1020                         WARN_ON(!pasync_handle);
1021                         pasync_handle->consumed = 0;
1022
1023                         pfree_link = pfree_link->next;
1024
1025                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1026                                                 is_header, host_write_num);
1027
1028                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1029                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1030
1031                         list_move(&pasync_handle->link, pbusy_list);
1032
1033                         host_write_num++;
1034                         host_write_num = host_write_num % num_entries;
1035                 }
1036
1037                 if (is_header) {
1038                         pasync_ctx->async_header.host_write_ptr =
1039                                                         host_write_num;
1040                         pasync_ctx->async_header.free_entries -= writables;
1041                         pasync_ctx->async_header.writables -= writables;
1042                         pasync_ctx->async_header.busy_entries += writables;
1043                 } else {
1044                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1045                         pasync_ctx->async_data.free_entries -= writables;
1046                         pasync_ctx->async_data.writables -= writables;
1047                         pasync_ctx->async_data.busy_entries += writables;
1048                 }
1049
1050                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1051                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1052                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1053                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1054                                         << DB_DEF_PDU_CQPROC_SHIFT;
1055
1056                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1057         }
1058 }
1059
1060 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1061                                          struct beiscsi_conn *beiscsi_conn,
1062                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1063 {
1064         struct hwi_controller *phwi_ctrlr;
1065         struct hwi_async_pdu_context *pasync_ctx;
1066         struct async_pdu_handle *pasync_handle = NULL;
1067         unsigned int cq_index = -1;
1068
1069         phwi_ctrlr = phba->phwi_ctrlr;
1070         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1071
1072         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1073                                              pdpdu_cqe, &cq_index);
1074         BUG_ON(pasync_handle->is_header != 0);
1075         if (pasync_handle->consumed == 0)
1076                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1077                                            cq_index);
1078
1079         hwi_free_async_msg(phba, pasync_handle->cri);
1080         hwi_post_async_buffers(phba, pasync_handle->is_header);
1081 }
1082
1083 static unsigned int
1084 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1085                   struct beiscsi_hba *phba,
1086                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1087 {
1088         struct list_head *plist;
1089         struct async_pdu_handle *pasync_handle;
1090         void *phdr = NULL;
1091         unsigned int hdr_len = 0, buf_len = 0;
1092         unsigned int status, index = 0, offset = 0;
1093         void *pfirst_buffer = NULL;
1094         unsigned int num_buf = 0;
1095
1096         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1097
1098         list_for_each_entry(pasync_handle, plist, link) {
1099                 if (index == 0) {
1100                         phdr = pasync_handle->pbuffer;
1101                         hdr_len = pasync_handle->buffer_len;
1102                 } else {
1103                         buf_len = pasync_handle->buffer_len;
1104                         if (!num_buf) {
1105                                 pfirst_buffer = pasync_handle->pbuffer;
1106                                 num_buf++;
1107                         }
1108                         memcpy(pfirst_buffer + offset,
1109                                pasync_handle->pbuffer, buf_len);
1110                         offset = buf_len;
1111                 }
1112                 index++;
1113         }
1114
1115         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1116                                            beiscsi_conn->beiscsi_conn_cid,
1117                                            phdr, hdr_len, pfirst_buffer,
1118                                            buf_len);
1119
1120         if (status == 0)
1121                 hwi_free_async_msg(phba, cri);
1122         return 0;
1123 }
1124
1125 static unsigned int
1126 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1127                      struct beiscsi_hba *phba,
1128                      struct async_pdu_handle *pasync_handle)
1129 {
1130         struct hwi_async_pdu_context *pasync_ctx;
1131         struct hwi_controller *phwi_ctrlr;
1132         unsigned int bytes_needed = 0, status = 0;
1133         unsigned short cri = pasync_handle->cri;
1134         struct pdu_base *ppdu;
1135
1136         phwi_ctrlr = phba->phwi_ctrlr;
1137         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1138
1139         list_del(&pasync_handle->link);
1140         if (pasync_handle->is_header) {
1141                 pasync_ctx->async_header.busy_entries--;
1142                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1143                         hwi_free_async_msg(phba, cri);
1144                         BUG();
1145                 }
1146
1147                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1148                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1149                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1150                                 (unsigned short)pasync_handle->buffer_len;
1151                 list_add_tail(&pasync_handle->link,
1152                               &pasync_ctx->async_entry[cri].wait_queue.list);
1153
1154                 ppdu = pasync_handle->pbuffer;
1155                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1156                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1157                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1158                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1159                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1160
1161                 if (status == 0) {
1162                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1163                             bytes_needed;
1164
1165                         if (bytes_needed == 0)
1166                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1167                                                            pasync_ctx, cri);
1168                 }
1169         } else {
1170                 pasync_ctx->async_data.busy_entries--;
1171                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1172                         list_add_tail(&pasync_handle->link,
1173                                       &pasync_ctx->async_entry[cri].wait_queue.
1174                                       list);
1175                         pasync_ctx->async_entry[cri].wait_queue.
1176                                 bytes_received +=
1177                                 (unsigned short)pasync_handle->buffer_len;
1178
1179                         if (pasync_ctx->async_entry[cri].wait_queue.
1180                             bytes_received >=
1181                             pasync_ctx->async_entry[cri].wait_queue.
1182                             bytes_needed)
1183                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1184                                                            pasync_ctx, cri);
1185                 }
1186         }
1187         return status;
1188 }
1189
1190 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1191                                          struct beiscsi_hba *phba,
1192                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1193 {
1194         struct hwi_controller *phwi_ctrlr;
1195         struct hwi_async_pdu_context *pasync_ctx;
1196         struct async_pdu_handle *pasync_handle = NULL;
1197         unsigned int cq_index = -1;
1198
1199         phwi_ctrlr = phba->phwi_ctrlr;
1200         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1201         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1202                                              pdpdu_cqe, &cq_index);
1203
1204         if (pasync_handle->consumed == 0)
1205                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1206                                            cq_index);
1207         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1208         hwi_post_async_buffers(phba, pasync_handle->is_header);
1209 }
1210
1211 static unsigned int beiscsi_process_cq(struct beiscsi_hba *phba)
1212 {
1213         struct hwi_controller *phwi_ctrlr;
1214         struct hwi_context_memory *phwi_context;
1215         struct be_queue_info *cq;
1216         struct sol_cqe *sol;
1217         struct dmsg_cqe *dmsg;
1218         unsigned int num_processed = 0;
1219         unsigned int tot_nump = 0;
1220         struct beiscsi_conn *beiscsi_conn;
1221
1222         phwi_ctrlr = phba->phwi_ctrlr;
1223         phwi_context = phwi_ctrlr->phwi_ctxt;
1224         cq = &phwi_context->be_cq;
1225         sol = queue_tail_node(cq);
1226
1227         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1228                CQE_VALID_MASK) {
1229                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1230
1231                 beiscsi_conn = phba->conn_table[(u32) (sol->
1232                                  dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1233                                  SOL_CID_MASK) >> 6];
1234
1235                 if (!beiscsi_conn || !beiscsi_conn->ep) {
1236                         shost_printk(KERN_WARNING, phba->shost,
1237                                      "Connection table empty for cid = %d\n",
1238                                      (u32)(sol->dw[offsetof(struct amap_sol_cqe,
1239                                      cid) / 32] & SOL_CID_MASK) >> 6);
1240                         return 0;
1241                 }
1242
1243                 if (num_processed >= 32) {
1244                         hwi_ring_cq_db(phba, phwi_context->be_cq.id,
1245                                         num_processed, 0, 0);
1246                         tot_nump += num_processed;
1247                         num_processed = 0;
1248                 }
1249
1250                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1251                         32] & CQE_CODE_MASK) {
1252                 case SOL_CMD_COMPLETE:
1253                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1254                         break;
1255                 case DRIVERMSG_NOTIFY:
1256                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1257                         dmsg = (struct dmsg_cqe *)sol;
1258                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1259                         break;
1260                 case UNSOL_HDR_NOTIFY:
1261                 case UNSOL_DATA_NOTIFY:
1262                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR/DATA_NOTIFY\n");
1263                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1264                                              (struct i_t_dpdu_cqe *)sol);
1265                         break;
1266                 case CXN_INVALIDATE_INDEX_NOTIFY:
1267                 case CMD_INVALIDATED_NOTIFY:
1268                 case CXN_INVALIDATE_NOTIFY:
1269                         SE_DEBUG(DBG_LVL_1,
1270                                  "Ignoring CQ Error notification for cmd/cxn"
1271                                  "invalidate\n");
1272                         break;
1273                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1274                 case CMD_KILLED_INVALID_STATSN_RCVD:
1275                 case CMD_KILLED_INVALID_R2T_RCVD:
1276                 case CMD_CXN_KILLED_LUN_INVALID:
1277                 case CMD_CXN_KILLED_ICD_INVALID:
1278                 case CMD_CXN_KILLED_ITT_INVALID:
1279                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1280                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1281                         SE_DEBUG(DBG_LVL_1,
1282                                  "CQ Error notification for cmd.. "
1283                                  "code %d cid 0x%x\n",
1284                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1285                                  32] & CQE_CODE_MASK,
1286                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1287                                  32] & SOL_CID_MASK));
1288                         break;
1289                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1290                         SE_DEBUG(DBG_LVL_1,
1291                                  "Digest error on def pdu ring, dropping..\n");
1292                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1293                                              (struct i_t_dpdu_cqe *) sol);
1294                         break;
1295                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1296                 case CXN_KILLED_BURST_LEN_MISMATCH:
1297                 case CXN_KILLED_AHS_RCVD:
1298                 case CXN_KILLED_HDR_DIGEST_ERR:
1299                 case CXN_KILLED_UNKNOWN_HDR:
1300                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1301                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1302                 case CXN_KILLED_TIMED_OUT:
1303                 case CXN_KILLED_FIN_RCVD:
1304                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1305                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1306                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1307                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1308                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1309                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, resetting CID "
1310                                  "0x%x...\n",
1311                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1312                                  32] & CQE_CODE_MASK,
1313                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1314                                  32] & CQE_CID_MASK);
1315                         iscsi_conn_failure(beiscsi_conn->conn,
1316                                            ISCSI_ERR_CONN_FAILED);
1317                         break;
1318                 case CXN_KILLED_RST_SENT:
1319                 case CXN_KILLED_RST_RCVD:
1320                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset received/sent "
1321                                  "on CID 0x%x...\n",
1322                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1323                                  32] & CQE_CODE_MASK,
1324                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1325                                  32] & CQE_CID_MASK);
1326                         iscsi_conn_failure(beiscsi_conn->conn,
1327                                            ISCSI_ERR_CONN_FAILED);
1328                         break;
1329                 default:
1330                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1331                                  "received on CID 0x%x...\n",
1332                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1333                                  32] & CQE_CODE_MASK,
1334                                  sol->dw[offsetof(struct amap_sol_cqe, cid) /
1335                                  32] & CQE_CID_MASK);
1336                         break;
1337                 }
1338
1339                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1340                 queue_tail_inc(cq);
1341                 sol = queue_tail_node(cq);
1342                 num_processed++;
1343         }
1344
1345         if (num_processed > 0) {
1346                 tot_nump += num_processed;
1347                 hwi_ring_cq_db(phba, phwi_context->be_cq.id, num_processed,
1348                                1, 0);
1349         }
1350         return tot_nump;
1351 }
1352
1353 static void beiscsi_process_all_cqs(struct work_struct *work)
1354 {
1355         unsigned long flags;
1356         struct beiscsi_hba *phba =
1357             container_of(work, struct beiscsi_hba, work_cqs);
1358
1359         if (phba->todo_mcc_cq) {
1360                 spin_lock_irqsave(&phba->isr_lock, flags);
1361                 phba->todo_mcc_cq = 0;
1362                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1363                 SE_DEBUG(DBG_LVL_1, "MCC Interrupt Not expected \n");
1364         }
1365
1366         if (phba->todo_cq) {
1367                 spin_lock_irqsave(&phba->isr_lock, flags);
1368                 phba->todo_cq = 0;
1369                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1370                 beiscsi_process_cq(phba);
1371         }
1372 }
1373
1374 static int be_iopoll(struct blk_iopoll *iop, int budget)
1375 {
1376         static unsigned int ret;
1377         struct beiscsi_hba *phba;
1378
1379         phba = container_of(iop, struct beiscsi_hba, iopoll);
1380
1381         ret = beiscsi_process_cq(phba);
1382         if (ret < budget) {
1383                 struct hwi_controller *phwi_ctrlr;
1384                 struct hwi_context_memory *phwi_context;
1385
1386                 phwi_ctrlr = phba->phwi_ctrlr;
1387                 phwi_context = phwi_ctrlr->phwi_ctxt;
1388                 blk_iopoll_complete(iop);
1389                 hwi_ring_eq_db(phba, phwi_context->be_eq.q.id, 0,
1390                                                         0, 1, 1);
1391         }
1392         return ret;
1393 }
1394
1395 static void
1396 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1397               unsigned int num_sg, struct beiscsi_io_task *io_task)
1398 {
1399         struct iscsi_sge *psgl;
1400         unsigned short sg_len, index;
1401         unsigned int sge_len = 0;
1402         unsigned long long addr;
1403         struct scatterlist *l_sg;
1404         unsigned int offset;
1405
1406         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1407                                       io_task->bhs_pa.u.a32.address_lo);
1408         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1409                                       io_task->bhs_pa.u.a32.address_hi);
1410
1411         l_sg = sg;
1412         for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1413                 if (index == 0) {
1414                         sg_len = sg_dma_len(sg);
1415                         addr = (u64) sg_dma_address(sg);
1416                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1417                                                         (addr & 0xFFFFFFFF));
1418                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1419                                                         (addr >> 32));
1420                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1421                                                         sg_len);
1422                         sge_len = sg_len;
1423                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1424                                                         1);
1425                 } else {
1426                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1427                                                         0);
1428                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1429                                                         pwrb, sge_len);
1430                         sg_len = sg_dma_len(sg);
1431                         addr = (u64) sg_dma_address(sg);
1432                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1433                                                         (addr & 0xFFFFFFFF));
1434                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1435                                                         (addr >> 32));
1436                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1437                                                         sg_len);
1438                 }
1439         }
1440         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1441         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1442
1443         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1444
1445         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1446                         io_task->bhs_pa.u.a32.address_hi);
1447         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1448                         io_task->bhs_pa.u.a32.address_lo);
1449
1450         if (num_sg == 2)
1451                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1452         sg = l_sg;
1453         psgl++;
1454         psgl++;
1455         offset = 0;
1456         for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1457                 sg_len = sg_dma_len(sg);
1458                 addr = (u64) sg_dma_address(sg);
1459                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1460                                                 (addr & 0xFFFFFFFF));
1461                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1462                                                 (addr >> 32));
1463                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1464                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1465                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1466                 offset += sg_len;
1467         }
1468         psgl--;
1469         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1470 }
1471
1472 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1473 {
1474         struct iscsi_sge *psgl;
1475         unsigned long long addr;
1476         struct beiscsi_io_task *io_task = task->dd_data;
1477         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1478         struct beiscsi_hba *phba = beiscsi_conn->phba;
1479
1480         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1481         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1482                                 io_task->bhs_pa.u.a32.address_lo);
1483         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1484                                 io_task->bhs_pa.u.a32.address_hi);
1485
1486         if (task->data) {
1487                 if (task->data_count) {
1488                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1489                         addr = (u64) pci_map_single(phba->pcidev,
1490                                                     task->data,
1491                                                     task->data_count, 1);
1492                 } else {
1493                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1494                         addr = 0;
1495                 }
1496                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1497                                                 (addr & 0xFFFFFFFF));
1498                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1499                                                 (addr >> 32));
1500                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1501                                                 task->data_count);
1502
1503                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1504         } else {
1505                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1506                 addr = 0;
1507         }
1508
1509         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1510
1511         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1512
1513         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1514                       io_task->bhs_pa.u.a32.address_hi);
1515         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1516                       io_task->bhs_pa.u.a32.address_lo);
1517         if (task->data) {
1518                 psgl++;
1519                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1520                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1521                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1522                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1523                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1524                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1525
1526                 psgl++;
1527                 if (task->data) {
1528                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1529                                                 (addr & 0xFFFFFFFF));
1530                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1531                                                 (addr >> 32));
1532                 }
1533                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1534         }
1535         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1536 }
1537
1538 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1539 {
1540         unsigned int num_cq_pages, num_eq_pages, num_async_pdu_buf_pages;
1541         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1542         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1543
1544         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1545                                       sizeof(struct sol_cqe));
1546         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
1547                                       sizeof(struct be_eq_entry));
1548         num_async_pdu_buf_pages =
1549                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1550                                        phba->params.defpdu_hdr_sz);
1551         num_async_pdu_buf_sgl_pages =
1552                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1553                                        sizeof(struct phys_addr));
1554         num_async_pdu_data_pages =
1555                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1556                                        phba->params.defpdu_data_sz);
1557         num_async_pdu_data_sgl_pages =
1558                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1559                                        sizeof(struct phys_addr));
1560
1561         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1562
1563         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1564                                                  BE_ISCSI_PDU_HEADER_SIZE;
1565         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1566                                             sizeof(struct hwi_context_memory);
1567
1568         phba->mem_req[HWI_MEM_CQ] = num_cq_pages * PAGE_SIZE;
1569         phba->mem_req[HWI_MEM_EQ] = num_eq_pages * PAGE_SIZE;
1570
1571         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1572             * (phba->params.wrbs_per_cxn)
1573             * phba->params.cxns_per_ctrl;
1574         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1575                                  (phba->params.wrbs_per_cxn);
1576         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1577                                 phba->params.cxns_per_ctrl);
1578
1579         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1580                 phba->params.icds_per_ctrl;
1581         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1582                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1583
1584         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1585                 num_async_pdu_buf_pages * PAGE_SIZE;
1586         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1587                 num_async_pdu_data_pages * PAGE_SIZE;
1588         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1589                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1590         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1591                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1592         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1593                 phba->params.asyncpdus_per_ctrl *
1594                 sizeof(struct async_pdu_handle);
1595         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1596                 phba->params.asyncpdus_per_ctrl *
1597                 sizeof(struct async_pdu_handle);
1598         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1599                 sizeof(struct hwi_async_pdu_context) +
1600                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1601 }
1602
1603 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1604 {
1605         struct be_mem_descriptor *mem_descr;
1606         dma_addr_t bus_add;
1607         struct mem_array *mem_arr, *mem_arr_orig;
1608         unsigned int i, j, alloc_size, curr_alloc_size;
1609
1610         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1611         if (!phba->phwi_ctrlr)
1612                 return -ENOMEM;
1613
1614         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1615                                  GFP_KERNEL);
1616         if (!phba->init_mem) {
1617                 kfree(phba->phwi_ctrlr);
1618                 return -ENOMEM;
1619         }
1620
1621         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1622                                GFP_KERNEL);
1623         if (!mem_arr_orig) {
1624                 kfree(phba->init_mem);
1625                 kfree(phba->phwi_ctrlr);
1626                 return -ENOMEM;
1627         }
1628
1629         mem_descr = phba->init_mem;
1630         for (i = 0; i < SE_MEM_MAX; i++) {
1631                 j = 0;
1632                 mem_arr = mem_arr_orig;
1633                 alloc_size = phba->mem_req[i];
1634                 memset(mem_arr, 0, sizeof(struct mem_array) *
1635                        BEISCSI_MAX_FRAGS_INIT);
1636                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1637                 do {
1638                         mem_arr->virtual_address = pci_alloc_consistent(
1639                                                         phba->pcidev,
1640                                                         curr_alloc_size,
1641                                                         &bus_add);
1642                         if (!mem_arr->virtual_address) {
1643                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1644                                         goto free_mem;
1645                                 if (curr_alloc_size -
1646                                         rounddown_pow_of_two(curr_alloc_size))
1647                                         curr_alloc_size = rounddown_pow_of_two
1648                                                              (curr_alloc_size);
1649                                 else
1650                                         curr_alloc_size = curr_alloc_size / 2;
1651                         } else {
1652                                 mem_arr->bus_address.u.
1653                                     a64.address = (__u64) bus_add;
1654                                 mem_arr->size = curr_alloc_size;
1655                                 alloc_size -= curr_alloc_size;
1656                                 curr_alloc_size = min(be_max_phys_size *
1657                                                       1024, alloc_size);
1658                                 j++;
1659                                 mem_arr++;
1660                         }
1661                 } while (alloc_size);
1662                 mem_descr->num_elements = j;
1663                 mem_descr->size_in_bytes = phba->mem_req[i];
1664                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1665                                                GFP_KERNEL);
1666                 if (!mem_descr->mem_array)
1667                         goto free_mem;
1668
1669                 memcpy(mem_descr->mem_array, mem_arr_orig,
1670                        sizeof(struct mem_array) * j);
1671                 mem_descr++;
1672         }
1673         kfree(mem_arr_orig);
1674         return 0;
1675 free_mem:
1676         mem_descr->num_elements = j;
1677         while ((i) || (j)) {
1678                 for (j = mem_descr->num_elements; j > 0; j--) {
1679                         pci_free_consistent(phba->pcidev,
1680                                             mem_descr->mem_array[j - 1].size,
1681                                             mem_descr->mem_array[j - 1].
1682                                             virtual_address,
1683                                             mem_descr->mem_array[j - 1].
1684                                             bus_address.u.a64.address);
1685                 }
1686                 if (i) {
1687                         i--;
1688                         kfree(mem_descr->mem_array);
1689                         mem_descr--;
1690                 }
1691         }
1692         kfree(mem_arr_orig);
1693         kfree(phba->init_mem);
1694         kfree(phba->phwi_ctrlr);
1695         return -ENOMEM;
1696 }
1697
1698 static int beiscsi_get_memory(struct beiscsi_hba *phba)
1699 {
1700         beiscsi_find_mem_req(phba);
1701         return beiscsi_alloc_mem(phba);
1702 }
1703
1704 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1705 {
1706         struct pdu_data_out *pdata_out;
1707         struct pdu_nop_out *pnop_out;
1708         struct be_mem_descriptor *mem_descr;
1709
1710         mem_descr = phba->init_mem;
1711         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1712         pdata_out =
1713             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1714         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1715
1716         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1717                       IIOC_SCSI_DATA);
1718
1719         pnop_out =
1720             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1721                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1722
1723         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1724         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1725         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1726         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1727 }
1728
1729 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1730 {
1731         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1732         struct wrb_handle *pwrb_handle;
1733         struct hwi_controller *phwi_ctrlr;
1734         struct hwi_wrb_context *pwrb_context;
1735         struct iscsi_wrb *pwrb;
1736         unsigned int num_cxn_wrbh;
1737         unsigned int num_cxn_wrb, j, idx, index;
1738
1739         mem_descr_wrbh = phba->init_mem;
1740         mem_descr_wrbh += HWI_MEM_WRBH;
1741
1742         mem_descr_wrb = phba->init_mem;
1743         mem_descr_wrb += HWI_MEM_WRB;
1744
1745         idx = 0;
1746         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
1747         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
1748                         ((sizeof(struct wrb_handle)) *
1749                          phba->params.wrbs_per_cxn));
1750         phwi_ctrlr = phba->phwi_ctrlr;
1751
1752         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
1753                 pwrb_context = &phwi_ctrlr->wrb_context[index];
1754                 SE_DEBUG(DBG_LVL_8, "cid=%d pwrb_context=%p \n", index,
1755                                                 pwrb_context);
1756                 pwrb_context->pwrb_handle_base =
1757                                 kzalloc(sizeof(struct wrb_handle *) *
1758                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
1759                 pwrb_context->pwrb_handle_basestd =
1760                                 kzalloc(sizeof(struct wrb_handle *) *
1761                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
1762                 if (num_cxn_wrbh) {
1763                         pwrb_context->alloc_index = 0;
1764                         pwrb_context->wrb_handles_available = 0;
1765                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1766                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1767                                 pwrb_context->pwrb_handle_basestd[j] =
1768                                                                 pwrb_handle;
1769                                 pwrb_context->wrb_handles_available++;
1770                                 pwrb_handle++;
1771                         }
1772                         pwrb_context->free_index = 0;
1773                         num_cxn_wrbh--;
1774                 } else {
1775                         idx++;
1776                         pwrb_handle =
1777                             mem_descr_wrbh->mem_array[idx].virtual_address;
1778                         num_cxn_wrbh =
1779                             ((mem_descr_wrbh->mem_array[idx].size) /
1780                              ((sizeof(struct wrb_handle)) *
1781                               phba->params.wrbs_per_cxn));
1782                         pwrb_context->alloc_index = 0;
1783                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1784                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
1785                                 pwrb_context->pwrb_handle_basestd[j] =
1786                                     pwrb_handle;
1787                                 pwrb_context->wrb_handles_available++;
1788                                 pwrb_handle++;
1789                         }
1790                         pwrb_context->free_index = 0;
1791                         num_cxn_wrbh--;
1792                 }
1793         }
1794         idx = 0;
1795         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1796         num_cxn_wrb =
1797             ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
1798              phba->params.wrbs_per_cxn);
1799
1800         for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
1801                 pwrb_context = &phwi_ctrlr->wrb_context[index];
1802                 if (num_cxn_wrb) {
1803                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1804                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1805                                 pwrb_handle->pwrb = pwrb;
1806                                 pwrb++;
1807                         }
1808                         num_cxn_wrb--;
1809                 } else {
1810                         idx++;
1811                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
1812                         num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
1813                                         (sizeof(struct iscsi_wrb)) *
1814                                         phba->params.wrbs_per_cxn);
1815                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
1816                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
1817                                 pwrb_handle->pwrb = pwrb;
1818                                 pwrb++;
1819                         }
1820                         num_cxn_wrb--;
1821                 }
1822         }
1823 }
1824
1825 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
1826 {
1827         struct hwi_controller *phwi_ctrlr;
1828         struct hba_parameters *p = &phba->params;
1829         struct hwi_async_pdu_context *pasync_ctx;
1830         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
1831         unsigned int index;
1832         struct be_mem_descriptor *mem_descr;
1833
1834         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1835         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
1836
1837         phwi_ctrlr = phba->phwi_ctrlr;
1838         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
1839                                 mem_descr->mem_array[0].virtual_address;
1840         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
1841         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
1842
1843         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
1844         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
1845         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
1846         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
1847
1848         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1849         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
1850         if (mem_descr->mem_array[0].virtual_address) {
1851                 SE_DEBUG(DBG_LVL_8,
1852                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
1853                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1854         } else
1855                 shost_printk(KERN_WARNING, phba->shost,
1856                              "No Virtual address \n");
1857
1858         pasync_ctx->async_header.va_base =
1859                         mem_descr->mem_array[0].virtual_address;
1860
1861         pasync_ctx->async_header.pa_base.u.a64.address =
1862                         mem_descr->mem_array[0].bus_address.u.a64.address;
1863
1864         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1865         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
1866         if (mem_descr->mem_array[0].virtual_address) {
1867                 SE_DEBUG(DBG_LVL_8,
1868                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
1869                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1870         } else
1871                 shost_printk(KERN_WARNING, phba->shost,
1872                             "No Virtual address \n");
1873         pasync_ctx->async_header.ring_base =
1874                         mem_descr->mem_array[0].virtual_address;
1875
1876         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1877         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
1878         if (mem_descr->mem_array[0].virtual_address) {
1879                 SE_DEBUG(DBG_LVL_8,
1880                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
1881                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1882         } else
1883                 shost_printk(KERN_WARNING, phba->shost,
1884                             "No Virtual address \n");
1885
1886         pasync_ctx->async_header.handle_base =
1887                         mem_descr->mem_array[0].virtual_address;
1888         pasync_ctx->async_header.writables = 0;
1889         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
1890
1891         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1892         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
1893         if (mem_descr->mem_array[0].virtual_address) {
1894                 SE_DEBUG(DBG_LVL_8,
1895                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
1896                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1897         } else
1898                 shost_printk(KERN_WARNING, phba->shost,
1899                             "No Virtual address \n");
1900         pasync_ctx->async_data.va_base =
1901                         mem_descr->mem_array[0].virtual_address;
1902         pasync_ctx->async_data.pa_base.u.a64.address =
1903                         mem_descr->mem_array[0].bus_address.u.a64.address;
1904
1905         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1906         mem_descr += HWI_MEM_ASYNC_DATA_RING;
1907         if (mem_descr->mem_array[0].virtual_address) {
1908                 SE_DEBUG(DBG_LVL_8,
1909                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
1910                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
1911         } else
1912                 shost_printk(KERN_WARNING, phba->shost,
1913                              "No Virtual address \n");
1914
1915         pasync_ctx->async_data.ring_base =
1916                         mem_descr->mem_array[0].virtual_address;
1917
1918         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
1919         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
1920         if (!mem_descr->mem_array[0].virtual_address)
1921                 shost_printk(KERN_WARNING, phba->shost,
1922                             "No Virtual address \n");
1923
1924         pasync_ctx->async_data.handle_base =
1925                         mem_descr->mem_array[0].virtual_address;
1926         pasync_ctx->async_data.writables = 0;
1927         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
1928
1929         pasync_header_h =
1930                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
1931         pasync_data_h =
1932                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
1933
1934         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
1935                 pasync_header_h->cri = -1;
1936                 pasync_header_h->index = (char)index;
1937                 INIT_LIST_HEAD(&pasync_header_h->link);
1938                 pasync_header_h->pbuffer =
1939                         (void *)((unsigned long)
1940                         (pasync_ctx->async_header.va_base) +
1941                         (p->defpdu_hdr_sz * index));
1942
1943                 pasync_header_h->pa.u.a64.address =
1944                         pasync_ctx->async_header.pa_base.u.a64.address +
1945                         (p->defpdu_hdr_sz * index);
1946
1947                 list_add_tail(&pasync_header_h->link,
1948                                 &pasync_ctx->async_header.free_list);
1949                 pasync_header_h++;
1950                 pasync_ctx->async_header.free_entries++;
1951                 pasync_ctx->async_header.writables++;
1952
1953                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
1954                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
1955                                header_busy_list);
1956                 pasync_data_h->cri = -1;
1957                 pasync_data_h->index = (char)index;
1958                 INIT_LIST_HEAD(&pasync_data_h->link);
1959                 pasync_data_h->pbuffer =
1960                         (void *)((unsigned long)
1961                         (pasync_ctx->async_data.va_base) +
1962                         (p->defpdu_data_sz * index));
1963
1964                 pasync_data_h->pa.u.a64.address =
1965                     pasync_ctx->async_data.pa_base.u.a64.address +
1966                     (p->defpdu_data_sz * index);
1967
1968                 list_add_tail(&pasync_data_h->link,
1969                               &pasync_ctx->async_data.free_list);
1970                 pasync_data_h++;
1971                 pasync_ctx->async_data.free_entries++;
1972                 pasync_ctx->async_data.writables++;
1973
1974                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
1975         }
1976
1977         pasync_ctx->async_header.host_write_ptr = 0;
1978         pasync_ctx->async_header.ep_read_ptr = -1;
1979         pasync_ctx->async_data.host_write_ptr = 0;
1980         pasync_ctx->async_data.ep_read_ptr = -1;
1981 }
1982
1983 static int
1984 be_sgl_create_contiguous(void *virtual_address,
1985                          u64 physical_address, u32 length,
1986                          struct be_dma_mem *sgl)
1987 {
1988         WARN_ON(!virtual_address);
1989         WARN_ON(!physical_address);
1990         WARN_ON(!length > 0);
1991         WARN_ON(!sgl);
1992
1993         sgl->va = virtual_address;
1994         sgl->dma = physical_address;
1995         sgl->size = length;
1996
1997         return 0;
1998 }
1999
2000 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2001 {
2002         memset(sgl, 0, sizeof(*sgl));
2003 }
2004
2005 static void
2006 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2007                      struct mem_array *pmem, struct be_dma_mem *sgl)
2008 {
2009         if (sgl->va)
2010                 be_sgl_destroy_contiguous(sgl);
2011
2012         be_sgl_create_contiguous(pmem->virtual_address,
2013                                  pmem->bus_address.u.a64.address,
2014                                  pmem->size, sgl);
2015 }
2016
2017 static void
2018 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2019                            struct mem_array *pmem, struct be_dma_mem *sgl)
2020 {
2021         if (sgl->va)
2022                 be_sgl_destroy_contiguous(sgl);
2023
2024         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2025                                  pmem->bus_address.u.a64.address,
2026                                  pmem->size, sgl);
2027 }
2028
2029 static int be_fill_queue(struct be_queue_info *q,
2030                 u16 len, u16 entry_size, void *vaddress)
2031 {
2032         struct be_dma_mem *mem = &q->dma_mem;
2033
2034         memset(q, 0, sizeof(*q));
2035         q->len = len;
2036         q->entry_size = entry_size;
2037         mem->size = len * entry_size;
2038         mem->va = vaddress;
2039         if (!mem->va)
2040                 return -ENOMEM;
2041         memset(mem->va, 0, mem->size);
2042         return 0;
2043 }
2044
2045 static int beiscsi_create_eq(struct beiscsi_hba *phba,
2046                              struct hwi_context_memory *phwi_context)
2047 {
2048         unsigned int idx;
2049         int ret;
2050         struct be_queue_info *eq;
2051         struct be_dma_mem *mem;
2052         struct be_mem_descriptor *mem_descr;
2053         void *eq_vaddress;
2054
2055         idx = 0;
2056         eq = &phwi_context->be_eq.q;
2057         mem = &eq->dma_mem;
2058         mem_descr = phba->init_mem;
2059         mem_descr += HWI_MEM_EQ;
2060         eq_vaddress = mem_descr->mem_array[idx].virtual_address;
2061
2062         ret = be_fill_queue(eq, phba->params.num_eq_entries,
2063                             sizeof(struct be_eq_entry), eq_vaddress);
2064         if (ret) {
2065                 shost_printk(KERN_ERR, phba->shost,
2066                              "be_fill_queue Failed for EQ \n");
2067                 return ret;
2068         }
2069
2070         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2071
2072         ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2073                                     phwi_context->be_eq.cur_eqd);
2074         if (ret) {
2075                 shost_printk(KERN_ERR, phba->shost, "beiscsi_cmd_eq_create"
2076                              "Failedfor EQ \n");
2077                 return ret;
2078         }
2079         SE_DEBUG(DBG_LVL_8, "eq id is %d\n", phwi_context->be_eq.q.id);
2080         return 0;
2081 }
2082
2083 static int beiscsi_create_cq(struct beiscsi_hba *phba,
2084                              struct hwi_context_memory *phwi_context)
2085 {
2086         unsigned int idx;
2087         int ret;
2088         struct be_queue_info *cq, *eq;
2089         struct be_dma_mem *mem;
2090         struct be_mem_descriptor *mem_descr;
2091         void *cq_vaddress;
2092
2093         idx = 0;
2094         cq = &phwi_context->be_cq;
2095         eq = &phwi_context->be_eq.q;
2096         mem = &cq->dma_mem;
2097         mem_descr = phba->init_mem;
2098         mem_descr += HWI_MEM_CQ;
2099         cq_vaddress = mem_descr->mem_array[idx].virtual_address;
2100         ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2101                             sizeof(struct sol_cqe), cq_vaddress);
2102         if (ret) {
2103                 shost_printk(KERN_ERR, phba->shost,
2104                              "be_fill_queue Failed for ISCSI CQ \n");
2105                 return ret;
2106         }
2107
2108         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2109         ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, false, 0);
2110         if (ret) {
2111                 shost_printk(KERN_ERR, phba->shost,
2112                              "beiscsi_cmd_eq_create Failed for ISCSI CQ \n");
2113                 return ret;
2114         }
2115         SE_DEBUG(DBG_LVL_8, "iscsi cq id is %d\n", phwi_context->be_cq.id);
2116         SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2117         return 0;
2118 }
2119
2120 static int
2121 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2122                        struct hwi_context_memory *phwi_context,
2123                        struct hwi_controller *phwi_ctrlr,
2124                        unsigned int def_pdu_ring_sz)
2125 {
2126         unsigned int idx;
2127         int ret;
2128         struct be_queue_info *dq, *cq;
2129         struct be_dma_mem *mem;
2130         struct be_mem_descriptor *mem_descr;
2131         void *dq_vaddress;
2132
2133         idx = 0;
2134         dq = &phwi_context->be_def_hdrq;
2135         cq = &phwi_context->be_cq;
2136         mem = &dq->dma_mem;
2137         mem_descr = phba->init_mem;
2138         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2139         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2140         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2141                             sizeof(struct phys_addr),
2142                             sizeof(struct phys_addr), dq_vaddress);
2143         if (ret) {
2144                 shost_printk(KERN_ERR, phba->shost,
2145                              "be_fill_queue Failed for DEF PDU HDR\n");
2146                 return ret;
2147         }
2148         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2149         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2150                                               def_pdu_ring_sz,
2151                                               phba->params.defpdu_hdr_sz);
2152         if (ret) {
2153                 shost_printk(KERN_ERR, phba->shost,
2154                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2155                 return ret;
2156         }
2157         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2158         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2159                  phwi_context->be_def_hdrq.id);
2160         hwi_post_async_buffers(phba, 1);
2161         return 0;
2162 }
2163
2164 static int
2165 beiscsi_create_def_data(struct beiscsi_hba *phba,
2166                         struct hwi_context_memory *phwi_context,
2167                         struct hwi_controller *phwi_ctrlr,
2168                         unsigned int def_pdu_ring_sz)
2169 {
2170         unsigned int idx;
2171         int ret;
2172         struct be_queue_info *dataq, *cq;
2173         struct be_dma_mem *mem;
2174         struct be_mem_descriptor *mem_descr;
2175         void *dq_vaddress;
2176
2177         idx = 0;
2178         dataq = &phwi_context->be_def_dataq;
2179         cq = &phwi_context->be_cq;
2180         mem = &dataq->dma_mem;
2181         mem_descr = phba->init_mem;
2182         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2183         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2184         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2185                             sizeof(struct phys_addr),
2186                             sizeof(struct phys_addr), dq_vaddress);
2187         if (ret) {
2188                 shost_printk(KERN_ERR, phba->shost,
2189                              "be_fill_queue Failed for DEF PDU DATA\n");
2190                 return ret;
2191         }
2192         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2193         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2194                                               def_pdu_ring_sz,
2195                                               phba->params.defpdu_data_sz);
2196         if (ret) {
2197                 shost_printk(KERN_ERR, phba->shost,
2198                              "be_cmd_create_default_pdu_queue Failed"
2199                              " for DEF PDU DATA\n");
2200                 return ret;
2201         }
2202         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2203         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2204                  phwi_context->be_def_dataq.id);
2205         hwi_post_async_buffers(phba, 0);
2206         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2207         return 0;
2208 }
2209
2210 static int
2211 beiscsi_post_pages(struct beiscsi_hba *phba)
2212 {
2213         struct be_mem_descriptor *mem_descr;
2214         struct mem_array *pm_arr;
2215         unsigned int page_offset, i;
2216         struct be_dma_mem sgl;
2217         int status;
2218
2219         mem_descr = phba->init_mem;
2220         mem_descr += HWI_MEM_SGE;
2221         pm_arr = mem_descr->mem_array;
2222
2223         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2224                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2225         for (i = 0; i < mem_descr->num_elements; i++) {
2226                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2227                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2228                                                 page_offset,
2229                                                 (pm_arr->size / PAGE_SIZE));
2230                 page_offset += pm_arr->size / PAGE_SIZE;
2231                 if (status != 0) {
2232                         shost_printk(KERN_ERR, phba->shost,
2233                                      "post sgl failed.\n");
2234                         return status;
2235                 }
2236                 pm_arr++;
2237         }
2238         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2239         return 0;
2240 }
2241
2242 static int
2243 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2244                          struct hwi_context_memory *phwi_context,
2245                          struct hwi_controller *phwi_ctrlr)
2246 {
2247         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2248         u64 pa_addr_lo;
2249         unsigned int idx, num, i;
2250         struct mem_array *pwrb_arr;
2251         void *wrb_vaddr;
2252         struct be_dma_mem sgl;
2253         struct be_mem_descriptor *mem_descr;
2254         int status;
2255
2256         idx = 0;
2257         mem_descr = phba->init_mem;
2258         mem_descr += HWI_MEM_WRB;
2259         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2260                            GFP_KERNEL);
2261         if (!pwrb_arr) {
2262                 shost_printk(KERN_ERR, phba->shost,
2263                              "Memory alloc failed in create wrb ring.\n");
2264                 return -ENOMEM;
2265         }
2266         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2267         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2268         num_wrb_rings = mem_descr->mem_array[idx].size /
2269                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2270
2271         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2272                 if (num_wrb_rings) {
2273                         pwrb_arr[num].virtual_address = wrb_vaddr;
2274                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2275                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2276                                             sizeof(struct iscsi_wrb);
2277                         wrb_vaddr += pwrb_arr[num].size;
2278                         pa_addr_lo += pwrb_arr[num].size;
2279                         num_wrb_rings--;
2280                 } else {
2281                         idx++;
2282                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2283                         pa_addr_lo = mem_descr->mem_array[idx].\
2284                                         bus_address.u.a64.address;
2285                         num_wrb_rings = mem_descr->mem_array[idx].size /
2286                                         (phba->params.wrbs_per_cxn *
2287                                         sizeof(struct iscsi_wrb));
2288                         pwrb_arr[num].virtual_address = wrb_vaddr;
2289                         pwrb_arr[num].bus_address.u.a64.address\
2290                                                 = pa_addr_lo;
2291                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2292                                                  sizeof(struct iscsi_wrb);
2293                         wrb_vaddr += pwrb_arr[num].size;
2294                         pa_addr_lo   += pwrb_arr[num].size;
2295                         num_wrb_rings--;
2296                 }
2297         }
2298         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2299                 wrb_mem_index = 0;
2300                 offset = 0;
2301                 size = 0;
2302
2303                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2304                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2305                                             &phwi_context->be_wrbq[i]);
2306                 if (status != 0) {
2307                         shost_printk(KERN_ERR, phba->shost,
2308                                      "wrbq create failed.");
2309                         return status;
2310                 }
2311                 phwi_ctrlr->wrb_context[i].cid = phwi_context->be_wrbq[i].id;
2312         }
2313         kfree(pwrb_arr);
2314         return 0;
2315 }
2316
2317 static void free_wrb_handles(struct beiscsi_hba *phba)
2318 {
2319         unsigned int index;
2320         struct hwi_controller *phwi_ctrlr;
2321         struct hwi_wrb_context *pwrb_context;
2322
2323         phwi_ctrlr = phba->phwi_ctrlr;
2324         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2325                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2326                 kfree(pwrb_context->pwrb_handle_base);
2327                 kfree(pwrb_context->pwrb_handle_basestd);
2328         }
2329 }
2330
2331 static void hwi_cleanup(struct beiscsi_hba *phba)
2332 {
2333         struct be_queue_info *q;
2334         struct be_ctrl_info *ctrl = &phba->ctrl;
2335         struct hwi_controller *phwi_ctrlr;
2336         struct hwi_context_memory *phwi_context;
2337         int i;
2338
2339         phwi_ctrlr = phba->phwi_ctrlr;
2340         phwi_context = phwi_ctrlr->phwi_ctxt;
2341         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2342                 q = &phwi_context->be_wrbq[i];
2343                 if (q->created)
2344                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2345         }
2346
2347         free_wrb_handles(phba);
2348
2349         q = &phwi_context->be_def_hdrq;
2350         if (q->created)
2351                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2352
2353         q = &phwi_context->be_def_dataq;
2354         if (q->created)
2355                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2356
2357         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2358
2359         q = &phwi_context->be_cq;
2360         if (q->created)
2361                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2362
2363         q = &phwi_context->be_eq.q;
2364         if (q->created)
2365                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2366 }
2367
2368 static int hwi_init_port(struct beiscsi_hba *phba)
2369 {
2370         struct hwi_controller *phwi_ctrlr;
2371         struct hwi_context_memory *phwi_context;
2372         unsigned int def_pdu_ring_sz;
2373         struct be_ctrl_info *ctrl = &phba->ctrl;
2374         int status;
2375
2376         def_pdu_ring_sz =
2377                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2378         phwi_ctrlr = phba->phwi_ctrlr;
2379
2380         phwi_context = phwi_ctrlr->phwi_ctxt;
2381         phwi_context->be_eq.max_eqd = 0;
2382         phwi_context->be_eq.min_eqd = 0;
2383         phwi_context->be_eq.cur_eqd = 64;
2384         phwi_context->be_eq.enable_aic = false;
2385         be_cmd_fw_initialize(&phba->ctrl);
2386         status = beiscsi_create_eq(phba, phwi_context);
2387         if (status != 0) {
2388                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2389                 goto error;
2390         }
2391
2392         status = mgmt_check_supported_fw(ctrl);
2393         if (status != 0) {
2394                 shost_printk(KERN_ERR, phba->shost,
2395                              "Unsupported fw version \n");
2396                 goto error;
2397         }
2398
2399         status = mgmt_get_fw_config(ctrl, phba);
2400         if (status != 0) {
2401                 shost_printk(KERN_ERR, phba->shost,
2402                              "Error getting fw config\n");
2403                 goto error;
2404         }
2405
2406         status = beiscsi_create_cq(phba, phwi_context);
2407         if (status != 0) {
2408                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2409                 goto error;
2410         }
2411
2412         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2413                                         def_pdu_ring_sz);
2414         if (status != 0) {
2415                 shost_printk(KERN_ERR, phba->shost,
2416                              "Default Header not created\n");
2417                 goto error;
2418         }
2419
2420         status = beiscsi_create_def_data(phba, phwi_context,
2421                                          phwi_ctrlr, def_pdu_ring_sz);
2422         if (status != 0) {
2423                 shost_printk(KERN_ERR, phba->shost,
2424                              "Default Data not created\n");
2425                 goto error;
2426         }
2427
2428         status = beiscsi_post_pages(phba);
2429         if (status != 0) {
2430                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2431                 goto error;
2432         }
2433
2434         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2435         if (status != 0) {
2436                 shost_printk(KERN_ERR, phba->shost,
2437                              "WRB Rings not created\n");
2438                 goto error;
2439         }
2440
2441         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2442         return 0;
2443
2444 error:
2445         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2446         hwi_cleanup(phba);
2447         return -ENOMEM;
2448 }
2449
2450
2451 static int hwi_init_controller(struct beiscsi_hba *phba)
2452 {
2453         struct hwi_controller *phwi_ctrlr;
2454
2455         phwi_ctrlr = phba->phwi_ctrlr;
2456         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2457                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2458                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2459                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2460                          phwi_ctrlr->phwi_ctxt);
2461         } else {
2462                 shost_printk(KERN_ERR, phba->shost,
2463                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2464                              "Failing to load\n");
2465                 return -ENOMEM;
2466         }
2467
2468         iscsi_init_global_templates(phba);
2469         beiscsi_init_wrb_handle(phba);
2470         hwi_init_async_pdu_ctx(phba);
2471         if (hwi_init_port(phba) != 0) {
2472                 shost_printk(KERN_ERR, phba->shost,
2473                              "hwi_init_controller failed\n");
2474                 return -ENOMEM;
2475         }
2476         return 0;
2477 }
2478
2479 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2480 {
2481         struct be_mem_descriptor *mem_descr;
2482         int i, j;
2483
2484         mem_descr = phba->init_mem;
2485         i = 0;
2486         j = 0;
2487         for (i = 0; i < SE_MEM_MAX; i++) {
2488                 for (j = mem_descr->num_elements; j > 0; j--) {
2489                         pci_free_consistent(phba->pcidev,
2490                           mem_descr->mem_array[j - 1].size,
2491                           mem_descr->mem_array[j - 1].virtual_address,
2492                           mem_descr->mem_array[j - 1].bus_address.
2493                                 u.a64.address);
2494                 }
2495                 kfree(mem_descr->mem_array);
2496                 mem_descr++;
2497         }
2498         kfree(phba->init_mem);
2499         kfree(phba->phwi_ctrlr);
2500 }
2501
2502 static int beiscsi_init_controller(struct beiscsi_hba *phba)
2503 {
2504         int ret = -ENOMEM;
2505
2506         ret = beiscsi_get_memory(phba);
2507         if (ret < 0) {
2508                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2509                              "Failed in beiscsi_alloc_memory \n");
2510                 return ret;
2511         }
2512
2513         ret = hwi_init_controller(phba);
2514         if (ret)
2515                 goto free_init;
2516         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2517         return 0;
2518
2519 free_init:
2520         beiscsi_free_mem(phba);
2521         return -ENOMEM;
2522 }
2523
2524 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2525 {
2526         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2527         struct sgl_handle *psgl_handle;
2528         struct iscsi_sge *pfrag;
2529         unsigned int arr_index, i, idx;
2530
2531         phba->io_sgl_hndl_avbl = 0;
2532         phba->eh_sgl_hndl_avbl = 0;
2533         mem_descr_sglh = phba->init_mem;
2534         mem_descr_sglh += HWI_MEM_SGLH;
2535         if (1 == mem_descr_sglh->num_elements) {
2536                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2537                                                  phba->params.ios_per_ctrl,
2538                                                  GFP_KERNEL);
2539                 if (!phba->io_sgl_hndl_base) {
2540                         shost_printk(KERN_ERR, phba->shost,
2541                                      "Mem Alloc Failed. Failing to load\n");
2542                         return -ENOMEM;
2543                 }
2544                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2545                                                  (phba->params.icds_per_ctrl -
2546                                                  phba->params.ios_per_ctrl),
2547                                                  GFP_KERNEL);
2548                 if (!phba->eh_sgl_hndl_base) {
2549                         kfree(phba->io_sgl_hndl_base);
2550                         shost_printk(KERN_ERR, phba->shost,
2551                                      "Mem Alloc Failed. Failing to load\n");
2552                         return -ENOMEM;
2553                 }
2554         } else {
2555                 shost_printk(KERN_ERR, phba->shost,
2556                              "HWI_MEM_SGLH is more than one element."
2557                              "Failing to load\n");
2558                 return -ENOMEM;
2559         }
2560
2561         arr_index = 0;
2562         idx = 0;
2563         while (idx < mem_descr_sglh->num_elements) {
2564                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2565
2566                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2567                       sizeof(struct sgl_handle)); i++) {
2568                         if (arr_index < phba->params.ios_per_ctrl) {
2569                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2570                                 phba->io_sgl_hndl_avbl++;
2571                                 arr_index++;
2572                         } else {
2573                                 phba->eh_sgl_hndl_base[arr_index -
2574                                         phba->params.ios_per_ctrl] =
2575                                                                 psgl_handle;
2576                                 arr_index++;
2577                                 phba->eh_sgl_hndl_avbl++;
2578                         }
2579                         psgl_handle++;
2580                 }
2581                 idx++;
2582         }
2583         SE_DEBUG(DBG_LVL_8,
2584                  "phba->io_sgl_hndl_avbl=%d"
2585                  "phba->eh_sgl_hndl_avbl=%d \n",
2586                  phba->io_sgl_hndl_avbl,
2587                  phba->eh_sgl_hndl_avbl);
2588         mem_descr_sg = phba->init_mem;
2589         mem_descr_sg += HWI_MEM_SGE;
2590         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
2591                  mem_descr_sg->num_elements);
2592         arr_index = 0;
2593         idx = 0;
2594         while (idx < mem_descr_sg->num_elements) {
2595                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
2596
2597                 for (i = 0;
2598                      i < (mem_descr_sg->mem_array[idx].size) /
2599                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
2600                      i++) {
2601                         if (arr_index < phba->params.ios_per_ctrl)
2602                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
2603                         else
2604                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
2605                                                 phba->params.ios_per_ctrl];
2606                         psgl_handle->pfrag = pfrag;
2607                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
2608                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
2609                         pfrag += phba->params.num_sge_per_io;
2610                         psgl_handle->sgl_index =
2611                                 phba->fw_config.iscsi_cid_start + arr_index++;
2612                 }
2613                 idx++;
2614         }
2615         phba->io_sgl_free_index = 0;
2616         phba->io_sgl_alloc_index = 0;
2617         phba->eh_sgl_free_index = 0;
2618         phba->eh_sgl_alloc_index = 0;
2619         return 0;
2620 }
2621
2622 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
2623 {
2624         int i, new_cid;
2625
2626         phba->cid_array = kmalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
2627                                   GFP_KERNEL);
2628         if (!phba->cid_array) {
2629                 shost_printk(KERN_ERR, phba->shost,
2630                              "Failed to allocate memory in "
2631                              "hba_setup_cid_tbls\n");
2632                 return -ENOMEM;
2633         }
2634         phba->ep_array = kmalloc(sizeof(struct iscsi_endpoint *) *
2635                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
2636         if (!phba->ep_array) {
2637                 shost_printk(KERN_ERR, phba->shost,
2638                              "Failed to allocate memory in "
2639                              "hba_setup_cid_tbls \n");
2640                 kfree(phba->cid_array);
2641                 return -ENOMEM;
2642         }
2643         new_cid = phba->fw_config.iscsi_icd_start;
2644         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2645                 phba->cid_array[i] = new_cid;
2646                 new_cid += 2;
2647         }
2648         phba->avlbl_cids = phba->params.cxns_per_ctrl;
2649         return 0;
2650 }
2651
2652 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
2653 {
2654         struct be_ctrl_info *ctrl = &phba->ctrl;
2655         struct hwi_controller *phwi_ctrlr;
2656         struct hwi_context_memory *phwi_context;
2657         struct be_queue_info *eq;
2658         u8 __iomem *addr;
2659         u32 reg;
2660         u32 enabled;
2661
2662         phwi_ctrlr = phba->phwi_ctrlr;
2663         phwi_context = phwi_ctrlr->phwi_ctxt;
2664
2665         eq = &phwi_context->be_eq.q;
2666         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
2667                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
2668         reg = ioread32(addr);
2669         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
2670
2671         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2672         if (!enabled) {
2673                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2674                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
2675                 iowrite32(reg, addr);
2676                 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
2677
2678                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
2679         } else
2680                 shost_printk(KERN_WARNING, phba->shost,
2681                              "In hwi_enable_intr, Not Enabled \n");
2682         return true;
2683 }
2684
2685 static void hwi_disable_intr(struct beiscsi_hba *phba)
2686 {
2687         struct be_ctrl_info *ctrl = &phba->ctrl;
2688
2689         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
2690         u32 reg = ioread32(addr);
2691
2692         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2693         if (enabled) {
2694                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
2695                 iowrite32(reg, addr);
2696         } else
2697                 shost_printk(KERN_WARNING, phba->shost,
2698                              "In hwi_disable_intr, Already Disabled \n");
2699 }
2700
2701 static int beiscsi_init_port(struct beiscsi_hba *phba)
2702 {
2703         int ret;
2704
2705         ret = beiscsi_init_controller(phba);
2706         if (ret < 0) {
2707                 shost_printk(KERN_ERR, phba->shost,
2708                              "beiscsi_dev_probe - Failed in"
2709                              "beiscsi_init_controller \n");
2710                 return ret;
2711         }
2712         ret = beiscsi_init_sgl_handle(phba);
2713         if (ret < 0) {
2714                 shost_printk(KERN_ERR, phba->shost,
2715                              "beiscsi_dev_probe - Failed in"
2716                              "beiscsi_init_sgl_handle \n");
2717                 goto do_cleanup_ctrlr;
2718         }
2719
2720         if (hba_setup_cid_tbls(phba)) {
2721                 shost_printk(KERN_ERR, phba->shost,
2722                              "Failed in hba_setup_cid_tbls\n");
2723                 kfree(phba->io_sgl_hndl_base);
2724                 kfree(phba->eh_sgl_hndl_base);
2725                 goto do_cleanup_ctrlr;
2726         }
2727
2728         return ret;
2729
2730 do_cleanup_ctrlr:
2731         hwi_cleanup(phba);
2732         return ret;
2733 }
2734
2735 static void hwi_purge_eq(struct beiscsi_hba *phba)
2736 {
2737         struct hwi_controller *phwi_ctrlr;
2738         struct hwi_context_memory *phwi_context;
2739         struct be_queue_info *eq;
2740         struct be_eq_entry *eqe = NULL;
2741
2742         phwi_ctrlr = phba->phwi_ctrlr;
2743         phwi_context = phwi_ctrlr->phwi_ctxt;
2744         eq = &phwi_context->be_eq.q;
2745         eqe = queue_tail_node(eq);
2746
2747         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
2748                                                 & EQE_VALID_MASK) {
2749                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2750                 queue_tail_inc(eq);
2751                 eqe = queue_tail_node(eq);
2752         }
2753 }
2754
2755 static void beiscsi_clean_port(struct beiscsi_hba *phba)
2756 {
2757         unsigned char mgmt_status;
2758
2759         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
2760         if (mgmt_status)
2761                 shost_printk(KERN_WARNING, phba->shost,
2762                              "mgmt_epfw_cleanup FAILED \n");
2763         hwi_cleanup(phba);
2764         hwi_purge_eq(phba);
2765         kfree(phba->io_sgl_hndl_base);
2766         kfree(phba->eh_sgl_hndl_base);
2767         kfree(phba->cid_array);
2768         kfree(phba->ep_array);
2769 }
2770
2771 void
2772 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
2773                            struct beiscsi_offload_params *params)
2774 {
2775         struct wrb_handle *pwrb_handle;
2776         struct iscsi_target_context_update_wrb *pwrb = NULL;
2777         struct be_mem_descriptor *mem_descr;
2778         struct beiscsi_hba *phba = beiscsi_conn->phba;
2779         u32 doorbell = 0;
2780
2781         /*
2782          * We can always use 0 here because it is reserved by libiscsi for
2783          * login/startup related tasks.
2784          */
2785         pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid, 0);
2786         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
2787         memset(pwrb, 0, sizeof(*pwrb));
2788         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2789                       max_burst_length, pwrb, params->dw[offsetof
2790                       (struct amap_beiscsi_offload_params,
2791                       max_burst_length) / 32]);
2792         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2793                       max_send_data_segment_length, pwrb,
2794                       params->dw[offsetof(struct amap_beiscsi_offload_params,
2795                       max_send_data_segment_length) / 32]);
2796         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2797                       first_burst_length,
2798                       pwrb,
2799                       params->dw[offsetof(struct amap_beiscsi_offload_params,
2800                       first_burst_length) / 32]);
2801
2802         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
2803                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2804                       erl) / 32] & OFFLD_PARAMS_ERL));
2805         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
2806                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2807                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
2808         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
2809                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2810                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
2811         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
2812                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2813                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
2814         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
2815                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2816                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
2817         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
2818                       pwrb,
2819                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
2820                       exp_statsn) / 32] + 1));
2821         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
2822                       0x7);
2823         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
2824                       pwrb, pwrb_handle->wrb_index);
2825         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
2826                       pwrb, pwrb_handle->nxt_wrb_index);
2827         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2828                         session_state, pwrb, 0);
2829         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
2830                       pwrb, 1);
2831         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
2832                       pwrb, 0);
2833         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
2834                       0);
2835
2836         mem_descr = phba->init_mem;
2837         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2838
2839         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2840                         pad_buffer_addr_hi, pwrb,
2841                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
2842         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
2843                         pad_buffer_addr_lo, pwrb,
2844                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
2845
2846         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
2847
2848         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
2849         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) <<
2850                                         DB_DEF_PDU_WRB_INDEX_SHIFT;
2851         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
2852
2853         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
2854 }
2855
2856 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
2857                               int *index, int *age)
2858 {
2859         *index = be32_to_cpu(itt) >> 16;
2860         if (age)
2861                 *age = conn->session->age;
2862 }
2863
2864 /**
2865  * beiscsi_alloc_pdu - allocates pdu and related resources
2866  * @task: libiscsi task
2867  * @opcode: opcode of pdu for task
2868  *
2869  * This is called with the session lock held. It will allocate
2870  * the wrb and sgl if needed for the command. And it will prep
2871  * the pdu's itt. beiscsi_parse_pdu will later translate
2872  * the pdu itt to the libiscsi task itt.
2873  */
2874 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
2875 {
2876         struct beiscsi_io_task *io_task = task->dd_data;
2877         struct iscsi_conn *conn = task->conn;
2878         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2879         struct beiscsi_hba *phba = beiscsi_conn->phba;
2880         struct hwi_wrb_context *pwrb_context;
2881         struct hwi_controller *phwi_ctrlr;
2882         itt_t itt;
2883         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
2884         dma_addr_t paddr;
2885
2886         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
2887                                           GFP_KERNEL, &paddr);
2888
2889         if (!io_task->cmd_bhs)
2890                 return -ENOMEM;
2891
2892         io_task->bhs_pa.u.a64.address = paddr;
2893         io_task->pwrb_handle = alloc_wrb_handle(phba,
2894                                                 beiscsi_conn->beiscsi_conn_cid,
2895                                                 task->itt);
2896         io_task->pwrb_handle->pio_handle = task;
2897         io_task->conn = beiscsi_conn;
2898
2899         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
2900         task->hdr_max = sizeof(struct be_cmd_bhs);
2901
2902         if (task->sc) {
2903                 spin_lock(&phba->io_sgl_lock);
2904                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
2905                 spin_unlock(&phba->io_sgl_lock);
2906                 if (!io_task->psgl_handle)
2907                         goto free_hndls;
2908
2909         } else {
2910                 io_task->scsi_cmnd = NULL;
2911                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
2912                         if (!beiscsi_conn->login_in_progress) {
2913                                 spin_lock(&phba->mgmt_sgl_lock);
2914                                 io_task->psgl_handle = (struct sgl_handle *)
2915                                                 alloc_mgmt_sgl_handle(phba);
2916                                 spin_unlock(&phba->mgmt_sgl_lock);
2917                                 if (!io_task->psgl_handle)
2918                                         goto free_hndls;
2919
2920                                 beiscsi_conn->login_in_progress = 1;
2921                                 beiscsi_conn->plogin_sgl_handle =
2922                                                         io_task->psgl_handle;
2923                         } else {
2924                                 io_task->psgl_handle =
2925                                                 beiscsi_conn->plogin_sgl_handle;
2926                         }
2927                 } else {
2928                         spin_lock(&phba->mgmt_sgl_lock);
2929                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
2930                         spin_unlock(&phba->mgmt_sgl_lock);
2931                         if (!io_task->psgl_handle)
2932                                 goto free_hndls;
2933                 }
2934         }
2935         itt = (itt_t) cpu_to_be32(((unsigned int)task->itt << 16) |
2936                         (unsigned int)(io_task->psgl_handle->sgl_index));
2937         io_task->cmd_bhs->iscsi_hdr.itt = itt;
2938         return 0;
2939
2940 free_hndls:
2941         phwi_ctrlr = phba->phwi_ctrlr;
2942         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
2943         free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2944         io_task->pwrb_handle = NULL;
2945         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
2946                       io_task->bhs_pa.u.a64.address);
2947         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
2948         return -ENOMEM;
2949 }
2950
2951 static void beiscsi_cleanup_task(struct iscsi_task *task)
2952 {
2953         struct beiscsi_io_task *io_task = task->dd_data;
2954         struct iscsi_conn *conn = task->conn;
2955         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
2956         struct beiscsi_hba *phba = beiscsi_conn->phba;
2957         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
2958         struct hwi_wrb_context *pwrb_context;
2959         struct hwi_controller *phwi_ctrlr;
2960
2961         phwi_ctrlr = phba->phwi_ctrlr;
2962         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
2963         if (io_task->pwrb_handle) {
2964                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2965                 io_task->pwrb_handle = NULL;
2966         }
2967
2968         if (io_task->cmd_bhs) {
2969                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
2970                               io_task->bhs_pa.u.a64.address);
2971         }
2972
2973         if (task->sc) {
2974                 if (io_task->psgl_handle) {
2975                         spin_lock(&phba->io_sgl_lock);
2976                         free_io_sgl_handle(phba, io_task->psgl_handle);
2977                         spin_unlock(&phba->io_sgl_lock);
2978                         io_task->psgl_handle = NULL;
2979                 }
2980         } else {
2981                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
2982                         return;
2983                 if (io_task->psgl_handle) {
2984                         spin_lock(&phba->mgmt_sgl_lock);
2985                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
2986                         spin_unlock(&phba->mgmt_sgl_lock);
2987                         io_task->psgl_handle = NULL;
2988                 }
2989         }
2990 }
2991
2992 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
2993                           unsigned int num_sg, unsigned int xferlen,
2994                           unsigned int writedir)
2995 {
2996
2997         struct beiscsi_io_task *io_task = task->dd_data;
2998         struct iscsi_conn *conn = task->conn;
2999         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3000         struct beiscsi_hba *phba = beiscsi_conn->phba;
3001         struct iscsi_wrb *pwrb = NULL;
3002         unsigned int doorbell = 0;
3003
3004         pwrb = io_task->pwrb_handle->pwrb;
3005         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3006         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3007
3008         if (writedir) {
3009                 SE_DEBUG(DBG_LVL_4, " WRITE Command \t");
3010                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3011                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3012                               &io_task->cmd_bhs->iscsi_data_pdu,
3013                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3014                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3015                               &io_task->cmd_bhs->iscsi_data_pdu,
3016                               ISCSI_OPCODE_SCSI_DATA_OUT);
3017                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3018                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3019                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3020                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3021         } else {
3022                 SE_DEBUG(DBG_LVL_4, "READ Command \t");
3023                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3024                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3025         }
3026         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3027                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3028                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3029
3030         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3031                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3032                                   lun[0]));
3033         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3034         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3035                       io_task->pwrb_handle->wrb_index);
3036         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3037                       be32_to_cpu(task->cmdsn));
3038         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3039                       io_task->psgl_handle->sgl_index);
3040
3041         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3042
3043         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3044                       io_task->pwrb_handle->nxt_wrb_index);
3045         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3046
3047         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3048         doorbell |= (io_task->pwrb_handle->wrb_index &
3049                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3050         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3051
3052         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3053         return 0;
3054 }
3055
3056 static int beiscsi_mtask(struct iscsi_task *task)
3057 {
3058         struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3059         struct iscsi_conn *conn = task->conn;
3060         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3061         struct beiscsi_hba *phba = beiscsi_conn->phba;
3062         struct iscsi_wrb *pwrb = NULL;
3063         unsigned int doorbell = 0;
3064         struct iscsi_task *aborted_task;
3065
3066         pwrb = io_task->pwrb_handle->pwrb;
3067         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3068                       be32_to_cpu(task->cmdsn));
3069         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3070                       io_task->pwrb_handle->wrb_index);
3071         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3072                       io_task->psgl_handle->sgl_index);
3073
3074         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3075         case ISCSI_OP_LOGIN:
3076                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, TGT_DM_CMD);
3077                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3078                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3079                 hwi_write_buffer(pwrb, task);
3080                 break;
3081         case ISCSI_OP_NOOP_OUT:
3082                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_RD_CMD);
3083                 hwi_write_buffer(pwrb, task);
3084                 break;
3085         case ISCSI_OP_TEXT:
3086                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_WR_CMD);
3087                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3088                 hwi_write_buffer(pwrb, task);
3089                 break;
3090         case ISCSI_OP_SCSI_TMFUNC:
3091                 aborted_task = iscsi_itt_to_task(conn,
3092                                         ((struct iscsi_tm *)task->hdr)->rtt);
3093                  if (!aborted_task)
3094                         return 0;
3095                 aborted_io_task = aborted_task->dd_data;
3096                 if (!aborted_io_task->scsi_cmnd)
3097                         return 0;
3098
3099                 mgmt_invalidate_icds(phba,
3100                                      aborted_io_task->psgl_handle->sgl_index,
3101                                      beiscsi_conn->beiscsi_conn_cid);
3102                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, INI_TMF_CMD);
3103                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3104                 hwi_write_buffer(pwrb, task);
3105                 break;
3106         case ISCSI_OP_LOGOUT:
3107                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3108                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3109                                 HWH_TYPE_LOGOUT);
3110                 hwi_write_buffer(pwrb, task);
3111                 break;
3112
3113         default:
3114                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3115                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3116                 return -EINVAL;
3117         }
3118
3119         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3120                       be32_to_cpu(task->data_count));
3121         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3122                       io_task->pwrb_handle->nxt_wrb_index);
3123         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3124
3125         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3126         doorbell |= (io_task->pwrb_handle->wrb_index &
3127                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3128         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3129         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3130         return 0;
3131 }
3132
3133 static int beiscsi_task_xmit(struct iscsi_task *task)
3134 {
3135         struct iscsi_conn *conn = task->conn;
3136         struct beiscsi_io_task *io_task = task->dd_data;
3137         struct scsi_cmnd *sc = task->sc;
3138         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3139         struct scatterlist *sg;
3140         int num_sg;
3141         unsigned int  writedir = 0, xferlen = 0;
3142
3143         SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3144                  "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3145                  task, conn, beiscsi_conn);
3146         if (!sc)
3147                 return beiscsi_mtask(task);
3148
3149         io_task->scsi_cmnd = sc;
3150         num_sg = scsi_dma_map(sc);
3151         if (num_sg < 0) {
3152                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3153                 return num_sg;
3154         }
3155         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3156                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3157         xferlen = scsi_bufflen(sc);
3158         sg = scsi_sglist(sc);
3159         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3160                 writedir = 1;
3161                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3162                          task->imm_count);
3163         } else
3164                 writedir = 0;
3165         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3166 }
3167
3168 static void beiscsi_remove(struct pci_dev *pcidev)
3169 {
3170         struct beiscsi_hba *phba = NULL;
3171
3172         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3173         if (!phba) {
3174                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3175                 return;
3176         }
3177
3178         hwi_disable_intr(phba);
3179         if (phba->pcidev->irq)
3180                 free_irq(phba->pcidev->irq, phba);
3181         destroy_workqueue(phba->wq);
3182         if (blk_iopoll_enabled)
3183                 blk_iopoll_disable(&phba->iopoll);
3184
3185         beiscsi_clean_port(phba);
3186         beiscsi_free_mem(phba);
3187         beiscsi_unmap_pci_function(phba);
3188         pci_free_consistent(phba->pcidev,
3189                             phba->ctrl.mbox_mem_alloced.size,
3190                             phba->ctrl.mbox_mem_alloced.va,
3191                             phba->ctrl.mbox_mem_alloced.dma);
3192         iscsi_host_remove(phba->shost);
3193         pci_dev_put(phba->pcidev);
3194         iscsi_host_free(phba->shost);
3195 }
3196
3197 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3198                                 const struct pci_device_id *id)
3199 {
3200         struct beiscsi_hba *phba = NULL;
3201         int ret;
3202
3203         ret = beiscsi_enable_pci(pcidev);
3204         if (ret < 0) {
3205                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3206                              "Failed to enable pci device \n");
3207                 return ret;
3208         }
3209
3210         phba = beiscsi_hba_alloc(pcidev);
3211         if (!phba) {
3212                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3213                         " Failed in beiscsi_hba_alloc \n");
3214                 goto disable_pci;
3215         }
3216
3217         pci_set_drvdata(pcidev, phba);
3218         ret = be_ctrl_init(phba, pcidev);
3219         if (ret) {
3220                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3221                                 "Failed in be_ctrl_init\n");
3222                 goto hba_free;
3223         }
3224
3225         spin_lock_init(&phba->io_sgl_lock);
3226         spin_lock_init(&phba->mgmt_sgl_lock);
3227         spin_lock_init(&phba->isr_lock);
3228         beiscsi_get_params(phba);
3229         ret = beiscsi_init_port(phba);
3230         if (ret < 0) {
3231                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3232                              "Failed in beiscsi_init_port\n");
3233                 goto free_port;
3234         }
3235
3236         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3237                  phba->shost->host_no);
3238         phba->wq = create_singlethread_workqueue(phba->wq_name);
3239         if (!phba->wq) {
3240                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3241                                 "Failed to allocate work queue\n");
3242                 goto free_twq;
3243         }
3244
3245         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3246
3247         if (blk_iopoll_enabled) {
3248                 blk_iopoll_init(&phba->iopoll, be_iopoll_budget, be_iopoll);
3249                 blk_iopoll_enable(&phba->iopoll);
3250         }
3251
3252         ret = beiscsi_init_irqs(phba);
3253         if (ret < 0) {
3254                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3255                              "Failed to beiscsi_init_irqs\n");
3256                 goto free_blkenbld;
3257         }
3258         ret = hwi_enable_intr(phba);
3259         if (ret < 0) {
3260                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3261                              "Failed to hwi_enable_intr\n");
3262                 goto free_ctrlr;
3263         }
3264
3265         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3266         return 0;
3267
3268 free_ctrlr:
3269         if (phba->pcidev->irq)
3270                 free_irq(phba->pcidev->irq, phba);
3271 free_blkenbld:
3272         destroy_workqueue(phba->wq);
3273         if (blk_iopoll_enabled)
3274                 blk_iopoll_disable(&phba->iopoll);
3275 free_twq:
3276         beiscsi_clean_port(phba);
3277         beiscsi_free_mem(phba);
3278 free_port:
3279         pci_free_consistent(phba->pcidev,
3280                             phba->ctrl.mbox_mem_alloced.size,
3281                             phba->ctrl.mbox_mem_alloced.va,
3282                            phba->ctrl.mbox_mem_alloced.dma);
3283         beiscsi_unmap_pci_function(phba);
3284 hba_free:
3285         iscsi_host_remove(phba->shost);
3286         pci_dev_put(phba->pcidev);
3287         iscsi_host_free(phba->shost);
3288 disable_pci:
3289         pci_disable_device(pcidev);
3290         return ret;
3291 }
3292
3293 struct iscsi_transport beiscsi_iscsi_transport = {
3294         .owner = THIS_MODULE,
3295         .name = DRV_NAME,
3296         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3297                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3298         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3299                 ISCSI_MAX_XMIT_DLENGTH |
3300                 ISCSI_HDRDGST_EN |
3301                 ISCSI_DATADGST_EN |
3302                 ISCSI_INITIAL_R2T_EN |
3303                 ISCSI_MAX_R2T |
3304                 ISCSI_IMM_DATA_EN |
3305                 ISCSI_FIRST_BURST |
3306                 ISCSI_MAX_BURST |
3307                 ISCSI_PDU_INORDER_EN |
3308                 ISCSI_DATASEQ_INORDER_EN |
3309                 ISCSI_ERL |
3310                 ISCSI_CONN_PORT |
3311                 ISCSI_CONN_ADDRESS |
3312                 ISCSI_EXP_STATSN |
3313                 ISCSI_PERSISTENT_PORT |
3314                 ISCSI_PERSISTENT_ADDRESS |
3315                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3316                 ISCSI_USERNAME | ISCSI_PASSWORD |
3317                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3318                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3319                 ISCSI_LU_RESET_TMO |
3320                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3321                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3322         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3323                                 ISCSI_HOST_INITIATOR_NAME,
3324         .create_session = beiscsi_session_create,
3325         .destroy_session = beiscsi_session_destroy,
3326         .create_conn = beiscsi_conn_create,
3327         .bind_conn = beiscsi_conn_bind,
3328         .destroy_conn = iscsi_conn_teardown,
3329         .set_param = beiscsi_set_param,
3330         .get_conn_param = beiscsi_conn_get_param,
3331         .get_session_param = iscsi_session_get_param,
3332         .get_host_param = beiscsi_get_host_param,
3333         .start_conn = beiscsi_conn_start,
3334         .stop_conn = beiscsi_conn_stop,
3335         .send_pdu = iscsi_conn_send_pdu,
3336         .xmit_task = beiscsi_task_xmit,
3337         .cleanup_task = beiscsi_cleanup_task,
3338         .alloc_pdu = beiscsi_alloc_pdu,
3339         .parse_pdu_itt = beiscsi_parse_pdu,
3340         .get_stats = beiscsi_conn_get_stats,
3341         .ep_connect = beiscsi_ep_connect,
3342         .ep_poll = beiscsi_ep_poll,
3343         .ep_disconnect = beiscsi_ep_disconnect,
3344         .session_recovery_timedout = iscsi_session_recovery_timedout,
3345 };
3346
3347 static struct pci_driver beiscsi_pci_driver = {
3348         .name = DRV_NAME,
3349         .probe = beiscsi_dev_probe,
3350         .remove = beiscsi_remove,
3351         .id_table = beiscsi_pci_id_table
3352 };
3353
3354 static int __init beiscsi_module_init(void)
3355 {
3356         int ret;
3357
3358         beiscsi_scsi_transport =
3359                         iscsi_register_transport(&beiscsi_iscsi_transport);
3360         if (!beiscsi_scsi_transport) {
3361                 SE_DEBUG(DBG_LVL_1,
3362                          "beiscsi_module_init - Unable to  register beiscsi"
3363                          "transport.\n");
3364                 ret = -ENOMEM;
3365         }
3366         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3367                  &beiscsi_iscsi_transport);
3368
3369         ret = pci_register_driver(&beiscsi_pci_driver);
3370         if (ret) {
3371                 SE_DEBUG(DBG_LVL_1,
3372                          "beiscsi_module_init - Unable to  register"
3373                          "beiscsi pci driver.\n");
3374                 goto unregister_iscsi_transport;
3375         }
3376         return 0;
3377
3378 unregister_iscsi_transport:
3379         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3380         return ret;
3381 }
3382
3383 static void __exit beiscsi_module_exit(void)
3384 {
3385         pci_unregister_driver(&beiscsi_pci_driver);
3386         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3387 }
3388
3389 module_init(beiscsi_module_init);
3390 module_exit(beiscsi_module_exit);