Merge branches 'core', 'cxgb4', 'ipath', 'iser', 'lockdep', 'mlx4', 'nes', 'ocrdma...
Roland Dreier [Mon, 21 May 2012 16:00:47 +0000 (09:00 -0700)]
79 files changed:
MAINTAINERS
drivers/infiniband/Kconfig
drivers/infiniband/Makefile
drivers/infiniband/core/cma.c
drivers/infiniband/core/umem.c
drivers/infiniband/core/uverbs_cmd.c
drivers/infiniband/core/verbs.c
drivers/infiniband/hw/cxgb4/Makefile
drivers/infiniband/hw/cxgb4/cm.c
drivers/infiniband/hw/cxgb4/device.c
drivers/infiniband/hw/cxgb4/ev.c
drivers/infiniband/hw/cxgb4/id_table.c [new file with mode: 0644]
drivers/infiniband/hw/cxgb4/iw_cxgb4.h
drivers/infiniband/hw/cxgb4/mem.c
drivers/infiniband/hw/cxgb4/provider.c
drivers/infiniband/hw/cxgb4/qp.c
drivers/infiniband/hw/cxgb4/resource.c
drivers/infiniband/hw/cxgb4/t4.h
drivers/infiniband/hw/cxgb4/user.h
drivers/infiniband/hw/ipath/ipath_iba6110.c
drivers/infiniband/hw/ipath/ipath_intr.c
drivers/infiniband/hw/mlx4/cq.c
drivers/infiniband/hw/mlx4/main.c
drivers/infiniband/hw/mlx4/mlx4_ib.h
drivers/infiniband/hw/mlx4/mr.c
drivers/infiniband/hw/mlx4/qp.c
drivers/infiniband/hw/mlx4/srq.c
drivers/infiniband/hw/ocrdma/Kconfig [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/Makefile [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma.h [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_abi.h [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_ah.c [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_ah.h [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_hw.c [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_hw.h [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_main.c [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_sli.h [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_verbs.c [new file with mode: 0644]
drivers/infiniband/hw/ocrdma/ocrdma_verbs.h [new file with mode: 0644]
drivers/infiniband/hw/qib/qib.h
drivers/infiniband/hw/qib/qib_driver.c
drivers/infiniband/hw/qib/qib_iba6120.c
drivers/infiniband/hw/qib/qib_iba7220.c
drivers/infiniband/hw/qib/qib_iba7322.c
drivers/infiniband/hw/qib/qib_init.c
drivers/infiniband/hw/qib/qib_mad.c
drivers/infiniband/hw/qib/qib_qp.c
drivers/infiniband/hw/qib/qib_rc.c
drivers/infiniband/hw/qib/qib_ruc.c
drivers/infiniband/hw/qib/qib_sysfs.c
drivers/infiniband/hw/qib/qib_tx.c
drivers/infiniband/hw/qib/qib_uc.c
drivers/infiniband/hw/qib/qib_ud.c
drivers/infiniband/hw/qib/qib_verbs.h
drivers/infiniband/ulp/iser/iscsi_iser.c
drivers/infiniband/ulp/iser/iser_verbs.c
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
drivers/net/ethernet/chelsio/cxgb4/sge.c
drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
drivers/net/ethernet/emulex/benet/Makefile
drivers/net/ethernet/emulex/benet/be.h
drivers/net/ethernet/emulex/benet/be_cmds.c
drivers/net/ethernet/emulex/benet/be_cmds.h
drivers/net/ethernet/emulex/benet/be_hw.h
drivers/net/ethernet/emulex/benet/be_main.c
drivers/net/ethernet/emulex/benet/be_roce.c [new file with mode: 0644]
drivers/net/ethernet/emulex/benet/be_roce.h [new file with mode: 0644]
drivers/net/ethernet/mellanox/mlx4/alloc.c
drivers/net/ethernet/mellanox/mlx4/fw.c
drivers/net/ethernet/mellanox/mlx4/fw.h
drivers/net/ethernet/mellanox/mlx4/main.c
include/linux/mlx4/device.h
include/linux/mlx4/qp.h
include/rdma/ib_mad.h
include/rdma/ib_verbs.h

index 7071633..5e8a932 100644 (file)
@@ -3631,7 +3631,7 @@ S:        Maintained
 F:     drivers/net/ethernet/icplus/ipg.*
 
 IPATH DRIVER
-M:     Mike Marciniszyn <infinipath@qlogic.com>
+M:     Mike Marciniszyn <infinipath@intel.com>
 L:     linux-rdma@vger.kernel.org
 S:     Maintained
 F:     drivers/infiniband/hw/ipath/
@@ -5455,7 +5455,7 @@ L:        rtc-linux@googlegroups.com
 S:     Maintained
 
 QIB DRIVER
-M:     Mike Marciniszyn <infinipath@qlogic.com>
+M:     Mike Marciniszyn <infinipath@intel.com>
 L:     linux-rdma@vger.kernel.org
 S:     Supported
 F:     drivers/infiniband/hw/qib/
index eb0add3..a0f29c1 100644 (file)
@@ -51,6 +51,7 @@ source "drivers/infiniband/hw/cxgb3/Kconfig"
 source "drivers/infiniband/hw/cxgb4/Kconfig"
 source "drivers/infiniband/hw/mlx4/Kconfig"
 source "drivers/infiniband/hw/nes/Kconfig"
+source "drivers/infiniband/hw/ocrdma/Kconfig"
 
 source "drivers/infiniband/ulp/ipoib/Kconfig"
 
index a3b2d8e..bf846a1 100644 (file)
@@ -8,6 +8,7 @@ obj-$(CONFIG_INFINIBAND_CXGB3)          += hw/cxgb3/
 obj-$(CONFIG_INFINIBAND_CXGB4)         += hw/cxgb4/
 obj-$(CONFIG_MLX4_INFINIBAND)          += hw/mlx4/
 obj-$(CONFIG_INFINIBAND_NES)           += hw/nes/
+obj-$(CONFIG_INFINIBAND_OCRDMA)                += hw/ocrdma/
 obj-$(CONFIG_INFINIBAND_IPOIB)         += ulp/ipoib/
 obj-$(CONFIG_INFINIBAND_SRP)           += ulp/srp/
 obj-$(CONFIG_INFINIBAND_SRPT)          += ulp/srpt/
index e3e470f..79c7eeb 100644 (file)
@@ -1218,13 +1218,13 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
        }
        if (!conn_id) {
                ret = -ENOMEM;
-               goto out;
+               goto err1;
        }
 
        mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
        ret = cma_acquire_dev(conn_id);
        if (ret)
-               goto release_conn_id;
+               goto err2;
 
        conn_id->cm_id.ib = cm_id;
        cm_id->context = conn_id;
@@ -1236,31 +1236,33 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
         */
        atomic_inc(&conn_id->refcount);
        ret = conn_id->id.event_handler(&conn_id->id, &event);
-       if (!ret) {
-               /*
-                * Acquire mutex to prevent user executing rdma_destroy_id()
-                * while we're accessing the cm_id.
-                */
-               mutex_lock(&lock);
-               if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
-                       ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
-               mutex_unlock(&lock);
-               mutex_unlock(&conn_id->handler_mutex);
-               cma_deref_id(conn_id);
-               goto out;
-       }
+       if (ret)
+               goto err3;
+
+       /*
+        * Acquire mutex to prevent user executing rdma_destroy_id()
+        * while we're accessing the cm_id.
+        */
+       mutex_lock(&lock);
+       if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
+               ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
+       mutex_unlock(&lock);
+       mutex_unlock(&conn_id->handler_mutex);
+       mutex_unlock(&listen_id->handler_mutex);
        cma_deref_id(conn_id);
+       return 0;
 
+err3:
+       cma_deref_id(conn_id);
        /* Destroy the CM ID by returning a non-zero value. */
        conn_id->cm_id.ib = NULL;
-
-release_conn_id:
+err2:
        cma_exch(conn_id, RDMA_CM_DESTROYING);
        mutex_unlock(&conn_id->handler_mutex);
-       rdma_destroy_id(&conn_id->id);
-
-out:
+err1:
        mutex_unlock(&listen_id->handler_mutex);
+       if (conn_id)
+               rdma_destroy_id(&conn_id->id);
        return ret;
 }
 
index 71f0c0f..a841123 100644 (file)
@@ -269,7 +269,7 @@ void ib_umem_release(struct ib_umem *umem)
        } else
                down_write(&mm->mmap_sem);
 
-       current->mm->locked_vm -= diff;
+       current->mm->pinned_vm -= diff;
        up_write(&mm->mmap_sem);
        mmput(mm);
        kfree(umem);
index 4d27e4c..f9d0d7c 100644 (file)
 
 #include "uverbs.h"
 
-static struct lock_class_key pd_lock_key;
-static struct lock_class_key mr_lock_key;
-static struct lock_class_key cq_lock_key;
-static struct lock_class_key qp_lock_key;
-static struct lock_class_key ah_lock_key;
-static struct lock_class_key srq_lock_key;
-static struct lock_class_key xrcd_lock_key;
+struct uverbs_lock_class {
+       struct lock_class_key   key;
+       char                    name[16];
+};
+
+static struct uverbs_lock_class pd_lock_class  = { .name = "PD-uobj" };
+static struct uverbs_lock_class mr_lock_class  = { .name = "MR-uobj" };
+static struct uverbs_lock_class cq_lock_class  = { .name = "CQ-uobj" };
+static struct uverbs_lock_class qp_lock_class  = { .name = "QP-uobj" };
+static struct uverbs_lock_class ah_lock_class  = { .name = "AH-uobj" };
+static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" };
+static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" };
 
 #define INIT_UDATA(udata, ibuf, obuf, ilen, olen)                      \
        do {                                                            \
@@ -83,13 +88,13 @@ static struct lock_class_key xrcd_lock_key;
  */
 
 static void init_uobj(struct ib_uobject *uobj, u64 user_handle,
-                     struct ib_ucontext *context, struct lock_class_key *key)
+                     struct ib_ucontext *context, struct uverbs_lock_class *c)
 {
        uobj->user_handle = user_handle;
        uobj->context     = context;
        kref_init(&uobj->ref);
        init_rwsem(&uobj->mutex);
-       lockdep_set_class(&uobj->mutex, key);
+       lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name);
        uobj->live        = 0;
 }
 
@@ -522,7 +527,7 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
        if (!uobj)
                return -ENOMEM;
 
-       init_uobj(uobj, 0, file->ucontext, &pd_lock_key);
+       init_uobj(uobj, 0, file->ucontext, &pd_lock_class);
        down_write(&uobj->mutex);
 
        pd = file->device->ib_dev->alloc_pd(file->device->ib_dev,
@@ -750,7 +755,7 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
                goto err_tree_mutex_unlock;
        }
 
-       init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_key);
+       init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class);
 
        down_write(&obj->uobject.mutex);
 
@@ -947,7 +952,7 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
        if (!uobj)
                return -ENOMEM;
 
-       init_uobj(uobj, 0, file->ucontext, &mr_lock_key);
+       init_uobj(uobj, 0, file->ucontext, &mr_lock_class);
        down_write(&uobj->mutex);
 
        pd = idr_read_pd(cmd.pd_handle, file->ucontext);
@@ -1115,7 +1120,7 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
        if (!obj)
                return -ENOMEM;
 
-       init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_key);
+       init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_class);
        down_write(&obj->uobject.mutex);
 
        if (cmd.comp_channel >= 0) {
@@ -1399,6 +1404,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
        if (copy_from_user(&cmd, buf, sizeof cmd))
                return -EFAULT;
 
+       if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
+               return -EPERM;
+
        INIT_UDATA(&udata, buf + sizeof cmd,
                   (unsigned long) cmd.response + sizeof resp,
                   in_len - sizeof cmd, out_len - sizeof resp);
@@ -1407,7 +1415,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
        if (!obj)
                return -ENOMEM;
 
-       init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key);
+       init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class);
        down_write(&obj->uevent.uobject.mutex);
 
        if (cmd.qp_type == IB_QPT_XRC_TGT) {
@@ -1418,13 +1426,6 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
                }
                device = xrcd->device;
        } else {
-               pd  = idr_read_pd(cmd.pd_handle, file->ucontext);
-               scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
-               if (!pd || !scq) {
-                       ret = -EINVAL;
-                       goto err_put;
-               }
-
                if (cmd.qp_type == IB_QPT_XRC_INI) {
                        cmd.max_recv_wr = cmd.max_recv_sge = 0;
                } else {
@@ -1435,13 +1436,24 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
                                        goto err_put;
                                }
                        }
-                       rcq = (cmd.recv_cq_handle == cmd.send_cq_handle) ?
-                              scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
-                       if (!rcq) {
-                               ret = -EINVAL;
-                               goto err_put;
+
+                       if (cmd.recv_cq_handle != cmd.send_cq_handle) {
+                               rcq = idr_read_cq(cmd.recv_cq_handle, file->ucontext, 0);
+                               if (!rcq) {
+                                       ret = -EINVAL;
+                                       goto err_put;
+                               }
                        }
                }
+
+               scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, !!rcq);
+               rcq = rcq ?: scq;
+               pd  = idr_read_pd(cmd.pd_handle, file->ucontext);
+               if (!pd || !scq) {
+                       ret = -EINVAL;
+                       goto err_put;
+               }
+
                device = pd->device;
        }
 
@@ -1585,7 +1597,7 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
        if (!obj)
                return -ENOMEM;
 
-       init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key);
+       init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class);
        down_write(&obj->uevent.uobject.mutex);
 
        xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj);
@@ -2272,7 +2284,7 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
        if (!uobj)
                return -ENOMEM;
 
-       init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_key);
+       init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class);
        down_write(&uobj->mutex);
 
        pd = idr_read_pd(cmd.pd_handle, file->ucontext);
@@ -2476,30 +2488,30 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
        if (!obj)
                return -ENOMEM;
 
-       init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_key);
+       init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class);
        down_write(&obj->uevent.uobject.mutex);
 
-       pd  = idr_read_pd(cmd->pd_handle, file->ucontext);
-       if (!pd) {
-               ret = -EINVAL;
-               goto err;
-       }
-
        if (cmd->srq_type == IB_SRQT_XRC) {
-               attr.ext.xrc.cq  = idr_read_cq(cmd->cq_handle, file->ucontext, 0);
-               if (!attr.ext.xrc.cq) {
-                       ret = -EINVAL;
-                       goto err_put_pd;
-               }
-
                attr.ext.xrc.xrcd  = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj);
                if (!attr.ext.xrc.xrcd) {
                        ret = -EINVAL;
-                       goto err_put_cq;
+                       goto err;
                }
 
                obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
                atomic_inc(&obj->uxrcd->refcnt);
+
+               attr.ext.xrc.cq  = idr_read_cq(cmd->cq_handle, file->ucontext, 0);
+               if (!attr.ext.xrc.cq) {
+                       ret = -EINVAL;
+                       goto err_put_xrcd;
+               }
+       }
+
+       pd  = idr_read_pd(cmd->pd_handle, file->ucontext);
+       if (!pd) {
+               ret = -EINVAL;
+               goto err_put_cq;
        }
 
        attr.event_handler  = ib_uverbs_srq_event_handler;
@@ -2576,17 +2588,17 @@ err_destroy:
        ib_destroy_srq(srq);
 
 err_put:
-       if (cmd->srq_type == IB_SRQT_XRC) {
-               atomic_dec(&obj->uxrcd->refcnt);
-               put_uobj_read(xrcd_uobj);
-       }
+       put_pd_read(pd);
 
 err_put_cq:
        if (cmd->srq_type == IB_SRQT_XRC)
                put_cq_read(attr.ext.xrc.cq);
 
-err_put_pd:
-       put_pd_read(pd);
+err_put_xrcd:
+       if (cmd->srq_type == IB_SRQT_XRC) {
+               atomic_dec(&obj->uxrcd->refcnt);
+               put_uobj_read(xrcd_uobj);
+       }
 
 err:
        put_uobj_write(&obj->uevent.uobject);
index 575b780..30f199e 100644 (file)
@@ -479,6 +479,7 @@ static const struct {
                                [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
                                                IB_QP_PORT                      |
                                                IB_QP_QKEY),
+                               [IB_QPT_RAW_PACKET] = IB_QP_PORT,
                                [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
                                                IB_QP_PORT                      |
                                                IB_QP_ACCESS_FLAGS),
@@ -1183,23 +1184,33 @@ EXPORT_SYMBOL(ib_dealloc_fmr);
 
 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
 {
+       int ret;
+
        if (!qp->device->attach_mcast)
                return -ENOSYS;
        if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
                return -EINVAL;
 
-       return qp->device->attach_mcast(qp, gid, lid);
+       ret = qp->device->attach_mcast(qp, gid, lid);
+       if (!ret)
+               atomic_inc(&qp->usecnt);
+       return ret;
 }
 EXPORT_SYMBOL(ib_attach_mcast);
 
 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
 {
+       int ret;
+
        if (!qp->device->detach_mcast)
                return -ENOSYS;
        if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
                return -EINVAL;
 
-       return qp->device->detach_mcast(qp, gid, lid);
+       ret = qp->device->detach_mcast(qp, gid, lid);
+       if (!ret)
+               atomic_dec(&qp->usecnt);
+       return ret;
 }
 EXPORT_SYMBOL(ib_detach_mcast);
 
index 46b878c..e11cf72 100644 (file)
@@ -2,4 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
 
 obj-$(CONFIG_INFINIBAND_CXGB4) += iw_cxgb4.o
 
-iw_cxgb4-y :=  device.o cm.o provider.o mem.o cq.o qp.o resource.o ev.o
+iw_cxgb4-y :=  device.o cm.o provider.o mem.o cq.o qp.o resource.o ev.o id_table.o
index 92b4c2b..55ab284 100644 (file)
@@ -1362,7 +1362,10 @@ static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
 
        ep = lookup_tid(t, tid);
        PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
-       BUG_ON(!ep);
+       if (!ep) {
+               printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n");
+               return 0;
+       }
        mutex_lock(&ep->com.mutex);
        switch (ep->com.state) {
        case ABORTING:
@@ -1410,6 +1413,24 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
                return 0;
        }
 
+       /*
+        * Log interesting failures.
+        */
+       switch (status) {
+       case CPL_ERR_CONN_RESET:
+       case CPL_ERR_CONN_TIMEDOUT:
+               break;
+       default:
+               printk(KERN_INFO MOD "Active open failure - "
+                      "atid %u status %u errno %d %pI4:%u->%pI4:%u\n",
+                      atid, status, status2errno(status),
+                      &ep->com.local_addr.sin_addr.s_addr,
+                      ntohs(ep->com.local_addr.sin_port),
+                      &ep->com.remote_addr.sin_addr.s_addr,
+                      ntohs(ep->com.remote_addr.sin_port));
+               break;
+       }
+
        connect_reply_upcall(ep, status2errno(status));
        state_set(&ep->com, DEAD);
 
@@ -1593,7 +1614,7 @@ static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst,
                                        n, n->dev, 0);
                if (!ep->l2t)
                        goto out;
-               ep->mtu = dst_mtu(ep->dst);
+               ep->mtu = dst_mtu(dst);
                ep->tx_chan = cxgb4_port_chan(n->dev);
                ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1;
                step = cdev->rdev.lldi.ntxq /
@@ -2656,6 +2677,12 @@ static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb)
        unsigned int tid = GET_TID(req);
 
        ep = lookup_tid(t, tid);
+       if (!ep) {
+               printk(KERN_WARNING MOD
+                      "Abort on non-existent endpoint, tid %d\n", tid);
+               kfree_skb(skb);
+               return 0;
+       }
        if (is_neg_adv_abort(req->status)) {
                PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep,
                     ep->hwtid);
@@ -2667,11 +2694,8 @@ static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb)
 
        /*
         * Wake up any threads in rdma_init() or rdma_fini().
-        * However, this is not needed if com state is just
-        * MPA_REQ_SENT
         */
-       if (ep->com.state != MPA_REQ_SENT)
-               c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET);
+       c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET);
        sched(dev, skb);
        return 0;
 }
index 6d0df6e..cb4ecd7 100644 (file)
@@ -32,6 +32,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/debugfs.h>
+#include <linux/vmalloc.h>
 
 #include <rdma/ib_verbs.h>
 
@@ -44,6 +45,12 @@ MODULE_DESCRIPTION("Chelsio T4 RDMA Driver");
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRV_VERSION);
 
+struct uld_ctx {
+       struct list_head entry;
+       struct cxgb4_lld_info lldi;
+       struct c4iw_dev *dev;
+};
+
 static LIST_HEAD(uld_ctx_list);
 static DEFINE_MUTEX(dev_mutex);
 
@@ -115,7 +122,7 @@ static int qp_release(struct inode *inode, struct file *file)
                printk(KERN_INFO "%s null qpd?\n", __func__);
                return 0;
        }
-       kfree(qpd->buf);
+       vfree(qpd->buf);
        kfree(qpd);
        return 0;
 }
@@ -139,7 +146,7 @@ static int qp_open(struct inode *inode, struct file *file)
        spin_unlock_irq(&qpd->devp->lock);
 
        qpd->bufsize = count * 128;
-       qpd->buf = kmalloc(qpd->bufsize, GFP_KERNEL);
+       qpd->buf = vmalloc(qpd->bufsize);
        if (!qpd->buf) {
                ret = -ENOMEM;
                goto err1;
@@ -240,6 +247,81 @@ static const struct file_operations stag_debugfs_fops = {
        .llseek  = default_llseek,
 };
 
+static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY"};
+
+static int stats_show(struct seq_file *seq, void *v)
+{
+       struct c4iw_dev *dev = seq->private;
+
+       seq_printf(seq, "   Object: %10s %10s %10s %10s\n", "Total", "Current",
+                  "Max", "Fail");
+       seq_printf(seq, "     PDID: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur,
+                       dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail);
+       seq_printf(seq, "      QID: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur,
+                       dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail);
+       seq_printf(seq, "   TPTMEM: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur,
+                       dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail);
+       seq_printf(seq, "   PBLMEM: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur,
+                       dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail);
+       seq_printf(seq, "   RQTMEM: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur,
+                       dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail);
+       seq_printf(seq, "  OCQPMEM: %10llu %10llu %10llu %10llu\n",
+                       dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur,
+                       dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail);
+       seq_printf(seq, "  DB FULL: %10llu\n", dev->rdev.stats.db_full);
+       seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty);
+       seq_printf(seq, "  DB DROP: %10llu\n", dev->rdev.stats.db_drop);
+       seq_printf(seq, " DB State: %s Transitions %llu\n",
+                  db_state_str[dev->db_state],
+                  dev->rdev.stats.db_state_transitions);
+       return 0;
+}
+
+static int stats_open(struct inode *inode, struct file *file)
+{
+       return single_open(file, stats_show, inode->i_private);
+}
+
+static ssize_t stats_clear(struct file *file, const char __user *buf,
+               size_t count, loff_t *pos)
+{
+       struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private;
+
+       mutex_lock(&dev->rdev.stats.lock);
+       dev->rdev.stats.pd.max = 0;
+       dev->rdev.stats.pd.fail = 0;
+       dev->rdev.stats.qid.max = 0;
+       dev->rdev.stats.qid.fail = 0;
+       dev->rdev.stats.stag.max = 0;
+       dev->rdev.stats.stag.fail = 0;
+       dev->rdev.stats.pbl.max = 0;
+       dev->rdev.stats.pbl.fail = 0;
+       dev->rdev.stats.rqt.max = 0;
+       dev->rdev.stats.rqt.fail = 0;
+       dev->rdev.stats.ocqp.max = 0;
+       dev->rdev.stats.ocqp.fail = 0;
+       dev->rdev.stats.db_full = 0;
+       dev->rdev.stats.db_empty = 0;
+       dev->rdev.stats.db_drop = 0;
+       dev->rdev.stats.db_state_transitions = 0;
+       mutex_unlock(&dev->rdev.stats.lock);
+       return count;
+}
+
+static const struct file_operations stats_debugfs_fops = {
+       .owner   = THIS_MODULE,
+       .open    = stats_open,
+       .release = single_release,
+       .read    = seq_read,
+       .llseek  = seq_lseek,
+       .write   = stats_clear,
+};
+
 static int setup_debugfs(struct c4iw_dev *devp)
 {
        struct dentry *de;
@@ -256,6 +338,12 @@ static int setup_debugfs(struct c4iw_dev *devp)
                                 (void *)devp, &stag_debugfs_fops);
        if (de && de->d_inode)
                de->d_inode->i_size = 4096;
+
+       de = debugfs_create_file("stats", S_IWUSR, devp->debugfs_root,
+                       (void *)devp, &stats_debugfs_fops);
+       if (de && de->d_inode)
+               de->d_inode->i_size = 4096;
+
        return 0;
 }
 
@@ -269,9 +357,13 @@ void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
        list_for_each_safe(pos, nxt, &uctx->qpids) {
                entry = list_entry(pos, struct c4iw_qid_list, entry);
                list_del_init(&entry->entry);
-               if (!(entry->qid & rdev->qpmask))
-                       c4iw_put_resource(&rdev->resource.qid_fifo, entry->qid,
-                                         &rdev->resource.qid_fifo_lock);
+               if (!(entry->qid & rdev->qpmask)) {
+                       c4iw_put_resource(&rdev->resource.qid_table,
+                                         entry->qid);
+                       mutex_lock(&rdev->stats.lock);
+                       rdev->stats.qid.cur -= rdev->qpmask + 1;
+                       mutex_unlock(&rdev->stats.lock);
+               }
                kfree(entry);
        }
 
@@ -332,6 +424,13 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
                goto err1;
        }
 
+       rdev->stats.pd.total = T4_MAX_NUM_PD;
+       rdev->stats.stag.total = rdev->lldi.vr->stag.size;
+       rdev->stats.pbl.total = rdev->lldi.vr->pbl.size;
+       rdev->stats.rqt.total = rdev->lldi.vr->rq.size;
+       rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size;
+       rdev->stats.qid.total = rdev->lldi.vr->qp.size;
+
        err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD);
        if (err) {
                printk(KERN_ERR MOD "error %d initializing resources\n", err);
@@ -370,12 +469,6 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev)
        c4iw_destroy_resource(&rdev->resource);
 }
 
-struct uld_ctx {
-       struct list_head entry;
-       struct cxgb4_lld_info lldi;
-       struct c4iw_dev *dev;
-};
-
 static void c4iw_dealloc(struct uld_ctx *ctx)
 {
        c4iw_rdev_close(&ctx->dev->rdev);
@@ -440,6 +533,8 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
        idr_init(&devp->qpidr);
        idr_init(&devp->mmidr);
        spin_lock_init(&devp->lock);
+       mutex_init(&devp->rdev.stats.lock);
+       mutex_init(&devp->db_mutex);
 
        if (c4iw_debugfs_root) {
                devp->debugfs_root = debugfs_create_dir(
@@ -585,11 +680,234 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)
        return 0;
 }
 
+static int disable_qp_db(int id, void *p, void *data)
+{
+       struct c4iw_qp *qp = p;
+
+       t4_disable_wq_db(&qp->wq);
+       return 0;
+}
+
+static void stop_queues(struct uld_ctx *ctx)
+{
+       spin_lock_irq(&ctx->dev->lock);
+       if (ctx->dev->db_state == NORMAL) {
+               ctx->dev->rdev.stats.db_state_transitions++;
+               ctx->dev->db_state = FLOW_CONTROL;
+               idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL);
+       }
+       spin_unlock_irq(&ctx->dev->lock);
+}
+
+static int enable_qp_db(int id, void *p, void *data)
+{
+       struct c4iw_qp *qp = p;
+
+       t4_enable_wq_db(&qp->wq);
+       return 0;
+}
+
+static void resume_queues(struct uld_ctx *ctx)
+{
+       spin_lock_irq(&ctx->dev->lock);
+       if (ctx->dev->qpcnt <= db_fc_threshold &&
+           ctx->dev->db_state == FLOW_CONTROL) {
+               ctx->dev->db_state = NORMAL;
+               ctx->dev->rdev.stats.db_state_transitions++;
+               idr_for_each(&ctx->dev->qpidr, enable_qp_db, NULL);
+       }
+       spin_unlock_irq(&ctx->dev->lock);
+}
+
+struct qp_list {
+       unsigned idx;
+       struct c4iw_qp **qps;
+};
+
+static int add_and_ref_qp(int id, void *p, void *data)
+{
+       struct qp_list *qp_listp = data;
+       struct c4iw_qp *qp = p;
+
+       c4iw_qp_add_ref(&qp->ibqp);
+       qp_listp->qps[qp_listp->idx++] = qp;
+       return 0;
+}
+
+static int count_qps(int id, void *p, void *data)
+{
+       unsigned *countp = data;
+       (*countp)++;
+       return 0;
+}
+
+static void deref_qps(struct qp_list qp_list)
+{
+       int idx;
+
+       for (idx = 0; idx < qp_list.idx; idx++)
+               c4iw_qp_rem_ref(&qp_list.qps[idx]->ibqp);
+}
+
+static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
+{
+       int idx;
+       int ret;
+
+       for (idx = 0; idx < qp_list->idx; idx++) {
+               struct c4iw_qp *qp = qp_list->qps[idx];
+
+               ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0],
+                                         qp->wq.sq.qid,
+                                         t4_sq_host_wq_pidx(&qp->wq),
+                                         t4_sq_wq_size(&qp->wq));
+               if (ret) {
+                       printk(KERN_ERR MOD "%s: Fatal error - "
+                              "DB overflow recovery failed - "
+                              "error syncing SQ qid %u\n",
+                              pci_name(ctx->lldi.pdev), qp->wq.sq.qid);
+                       return;
+               }
+
+               ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0],
+                                         qp->wq.rq.qid,
+                                         t4_rq_host_wq_pidx(&qp->wq),
+                                         t4_rq_wq_size(&qp->wq));
+
+               if (ret) {
+                       printk(KERN_ERR MOD "%s: Fatal error - "
+                              "DB overflow recovery failed - "
+                              "error syncing RQ qid %u\n",
+                              pci_name(ctx->lldi.pdev), qp->wq.rq.qid);
+                       return;
+               }
+
+               /* Wait for the dbfifo to drain */
+               while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) {
+                       set_current_state(TASK_UNINTERRUPTIBLE);
+                       schedule_timeout(usecs_to_jiffies(10));
+               }
+       }
+}
+
+static void recover_queues(struct uld_ctx *ctx)
+{
+       int count = 0;
+       struct qp_list qp_list;
+       int ret;
+
+       /* lock out kernel db ringers */
+       mutex_lock(&ctx->dev->db_mutex);
+
+       /* put all queues in to recovery mode */
+       spin_lock_irq(&ctx->dev->lock);
+       ctx->dev->db_state = RECOVERY;
+       ctx->dev->rdev.stats.db_state_transitions++;
+       idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL);
+       spin_unlock_irq(&ctx->dev->lock);
+
+       /* slow everybody down */
+       set_current_state(TASK_UNINTERRUPTIBLE);
+       schedule_timeout(usecs_to_jiffies(1000));
+
+       /* Wait for the dbfifo to completely drain. */
+       while (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) > 0) {
+               set_current_state(TASK_UNINTERRUPTIBLE);
+               schedule_timeout(usecs_to_jiffies(10));
+       }
+
+       /* flush the SGE contexts */
+       ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]);
+       if (ret) {
+               printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n",
+                      pci_name(ctx->lldi.pdev));
+               goto out;
+       }
+
+       /* Count active queues so we can build a list of queues to recover */
+       spin_lock_irq(&ctx->dev->lock);
+       idr_for_each(&ctx->dev->qpidr, count_qps, &count);
+
+       qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC);
+       if (!qp_list.qps) {
+               printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n",
+                      pci_name(ctx->lldi.pdev));
+               spin_unlock_irq(&ctx->dev->lock);
+               goto out;
+       }
+       qp_list.idx = 0;
+
+       /* add and ref each qp so it doesn't get freed */
+       idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list);
+
+       spin_unlock_irq(&ctx->dev->lock);
+
+       /* now traverse the list in a safe context to recover the db state*/
+       recover_lost_dbs(ctx, &qp_list);
+
+       /* we're almost done!  deref the qps and clean up */
+       deref_qps(qp_list);
+       kfree(qp_list.qps);
+
+       /* Wait for the dbfifo to completely drain again */
+       while (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) > 0) {
+               set_current_state(TASK_UNINTERRUPTIBLE);
+               schedule_timeout(usecs_to_jiffies(10));
+       }
+
+       /* resume the queues */
+       spin_lock_irq(&ctx->dev->lock);
+       if (ctx->dev->qpcnt > db_fc_threshold)
+               ctx->dev->db_state = FLOW_CONTROL;
+       else {
+               ctx->dev->db_state = NORMAL;
+               idr_for_each(&ctx->dev->qpidr, enable_qp_db, NULL);
+       }
+       ctx->dev->rdev.stats.db_state_transitions++;
+       spin_unlock_irq(&ctx->dev->lock);
+
+out:
+       /* start up kernel db ringers again */
+       mutex_unlock(&ctx->dev->db_mutex);
+}
+
+static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...)
+{
+       struct uld_ctx *ctx = handle;
+
+       switch (control) {
+       case CXGB4_CONTROL_DB_FULL:
+               stop_queues(ctx);
+               mutex_lock(&ctx->dev->rdev.stats.lock);
+               ctx->dev->rdev.stats.db_full++;
+               mutex_unlock(&ctx->dev->rdev.stats.lock);
+               break;
+       case CXGB4_CONTROL_DB_EMPTY:
+               resume_queues(ctx);
+               mutex_lock(&ctx->dev->rdev.stats.lock);
+               ctx->dev->rdev.stats.db_empty++;
+               mutex_unlock(&ctx->dev->rdev.stats.lock);
+               break;
+       case CXGB4_CONTROL_DB_DROP:
+               recover_queues(ctx);
+               mutex_lock(&ctx->dev->rdev.stats.lock);
+               ctx->dev->rdev.stats.db_drop++;
+               mutex_unlock(&ctx->dev->rdev.stats.lock);
+               break;
+       default:
+               printk(KERN_WARNING MOD "%s: unknown control cmd %u\n",
+                      pci_name(ctx->lldi.pdev), control);
+               break;
+       }
+       return 0;
+}
+
 static struct cxgb4_uld_info c4iw_uld_info = {
        .name = DRV_NAME,
        .add = c4iw_uld_add,
        .rx_handler = c4iw_uld_rx_handler,
        .state_change = c4iw_uld_state_change,
+       .control = c4iw_uld_control,
 };
 
 static int __init c4iw_init_module(void)
index 397cb36..cf2f6b4 100644 (file)
@@ -84,7 +84,7 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
        struct c4iw_qp *qhp;
        u32 cqid;
 
-       spin_lock(&dev->lock);
+       spin_lock_irq(&dev->lock);
        qhp = get_qhp(dev, CQE_QPID(err_cqe));
        if (!qhp) {
                printk(KERN_ERR MOD "BAD AE qpid 0x%x opcode %d "
@@ -93,7 +93,7 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
                       CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
                       CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
                       CQE_WRID_LOW(err_cqe));
-               spin_unlock(&dev->lock);
+               spin_unlock_irq(&dev->lock);
                goto out;
        }
 
@@ -109,13 +109,13 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
                       CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
                       CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
                       CQE_WRID_LOW(err_cqe));
-               spin_unlock(&dev->lock);
+               spin_unlock_irq(&dev->lock);
                goto out;
        }
 
        c4iw_qp_add_ref(&qhp->ibqp);
        atomic_inc(&chp->refcnt);
-       spin_unlock(&dev->lock);
+       spin_unlock_irq(&dev->lock);
 
        /* Bad incoming write */
        if (RQ_TYPE(err_cqe) &&
diff --git a/drivers/infiniband/hw/cxgb4/id_table.c b/drivers/infiniband/hw/cxgb4/id_table.c
new file mode 100644 (file)
index 0000000..f95e5df
--- /dev/null
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2011 Chelsio Communications.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/kernel.h>
+#include <linux/random.h>
+#include "iw_cxgb4.h"
+
+#define RANDOM_SKIP 16
+
+/*
+ * Trivial bitmap-based allocator. If the random flag is set, the
+ * allocator is designed to:
+ * - pseudo-randomize the id returned such that it is not trivially predictable.
+ * - avoid reuse of recently used id (at the expense of predictability)
+ */
+u32 c4iw_id_alloc(struct c4iw_id_table *alloc)
+{
+       unsigned long flags;
+       u32 obj;
+
+       spin_lock_irqsave(&alloc->lock, flags);
+
+       obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last);
+       if (obj >= alloc->max)
+               obj = find_first_zero_bit(alloc->table, alloc->max);
+
+       if (obj < alloc->max) {
+               if (alloc->flags & C4IW_ID_TABLE_F_RANDOM)
+                       alloc->last += random32() % RANDOM_SKIP;
+               else
+                       alloc->last = obj + 1;
+               if (alloc->last >= alloc->max)
+                       alloc->last = 0;
+               set_bit(obj, alloc->table);
+               obj += alloc->start;
+       } else
+               obj = -1;
+
+       spin_unlock_irqrestore(&alloc->lock, flags);
+       return obj;
+}
+
+void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj)
+{
+       unsigned long flags;
+
+       obj -= alloc->start;
+       BUG_ON((int)obj < 0);
+
+       spin_lock_irqsave(&alloc->lock, flags);
+       clear_bit(obj, alloc->table);
+       spin_unlock_irqrestore(&alloc->lock, flags);
+}
+
+int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
+                       u32 reserved, u32 flags)
+{
+       int i;
+
+       alloc->start = start;
+       alloc->flags = flags;
+       if (flags & C4IW_ID_TABLE_F_RANDOM)
+               alloc->last = random32() % RANDOM_SKIP;
+       else
+               alloc->last = 0;
+       alloc->max  = num;
+       spin_lock_init(&alloc->lock);
+       alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof(long),
+                               GFP_KERNEL);
+       if (!alloc->table)
+               return -ENOMEM;
+
+       bitmap_zero(alloc->table, num);
+       if (!(alloc->flags & C4IW_ID_TABLE_F_EMPTY))
+               for (i = 0; i < reserved; ++i)
+                       set_bit(i, alloc->table);
+
+       return 0;
+}
+
+void c4iw_id_table_free(struct c4iw_id_table *alloc)
+{
+       kfree(alloc->table);
+}
index 1357c5b..9beb3a9 100644 (file)
@@ -45,7 +45,6 @@
 #include <linux/kref.h>
 #include <linux/timer.h>
 #include <linux/io.h>
-#include <linux/kfifo.h>
 
 #include <asm/byteorder.h>
 
@@ -79,13 +78,22 @@ static inline void *cplhdr(struct sk_buff *skb)
        return skb->data;
 }
 
+#define C4IW_ID_TABLE_F_RANDOM 1       /* Pseudo-randomize the id's returned */
+#define C4IW_ID_TABLE_F_EMPTY  2       /* Table is initially empty */
+
+struct c4iw_id_table {
+       u32 flags;
+       u32 start;              /* logical minimal id */
+       u32 last;               /* hint for find */
+       u32 max;
+       spinlock_t lock;
+       unsigned long *table;
+};
+
 struct c4iw_resource {
-       struct kfifo tpt_fifo;
-       spinlock_t tpt_fifo_lock;
-       struct kfifo qid_fifo;
-       spinlock_t qid_fifo_lock;
-       struct kfifo pdid_fifo;
-       spinlock_t pdid_fifo_lock;
+       struct c4iw_id_table tpt_table;
+       struct c4iw_id_table qid_table;
+       struct c4iw_id_table pdid_table;
 };
 
 struct c4iw_qid_list {
@@ -103,6 +111,27 @@ enum c4iw_rdev_flags {
        T4_FATAL_ERROR = (1<<0),
 };
 
+struct c4iw_stat {
+       u64 total;
+       u64 cur;
+       u64 max;
+       u64 fail;
+};
+
+struct c4iw_stats {
+       struct mutex lock;
+       struct c4iw_stat qid;
+       struct c4iw_stat pd;
+       struct c4iw_stat stag;
+       struct c4iw_stat pbl;
+       struct c4iw_stat rqt;
+       struct c4iw_stat ocqp;
+       u64  db_full;
+       u64  db_empty;
+       u64  db_drop;
+       u64  db_state_transitions;
+};
+
 struct c4iw_rdev {
        struct c4iw_resource resource;
        unsigned long qpshift;
@@ -117,6 +146,7 @@ struct c4iw_rdev {
        struct cxgb4_lld_info lldi;
        unsigned long oc_mw_pa;
        void __iomem *oc_mw_kva;
+       struct c4iw_stats stats;
 };
 
 static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
@@ -175,6 +205,12 @@ static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev,
        return wr_waitp->ret;
 }
 
+enum db_state {
+       NORMAL = 0,
+       FLOW_CONTROL = 1,
+       RECOVERY = 2
+};
+
 struct c4iw_dev {
        struct ib_device ibdev;
        struct c4iw_rdev rdev;
@@ -183,7 +219,10 @@ struct c4iw_dev {
        struct idr qpidr;
        struct idr mmidr;
        spinlock_t lock;
+       struct mutex db_mutex;
        struct dentry *debugfs_root;
+       enum db_state db_state;
+       int qpcnt;
 };
 
 static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
@@ -211,29 +250,57 @@ static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
        return idr_find(&rhp->mmidr, mmid);
 }
 
-static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
-                               void *handle, u32 id)
+static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr,
+                                void *handle, u32 id, int lock)
 {
        int ret;
        int newid;
 
        do {
-               if (!idr_pre_get(idr, GFP_KERNEL))
+               if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC))
                        return -ENOMEM;
-               spin_lock_irq(&rhp->lock);
+               if (lock)
+                       spin_lock_irq(&rhp->lock);
                ret = idr_get_new_above(idr, handle, id, &newid);
-               BUG_ON(newid != id);
-               spin_unlock_irq(&rhp->lock);
+               BUG_ON(!ret && newid != id);
+               if (lock)
+                       spin_unlock_irq(&rhp->lock);
        } while (ret == -EAGAIN);
 
        return ret;
 }
 
-static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
+static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
+                               void *handle, u32 id)
+{
+       return _insert_handle(rhp, idr, handle, id, 1);
+}
+
+static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr,
+                                      void *handle, u32 id)
+{
+       return _insert_handle(rhp, idr, handle, id, 0);
+}
+
+static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr,
+                                  u32 id, int lock)
 {
-       spin_lock_irq(&rhp->lock);
+       if (lock)
+               spin_lock_irq(&rhp->lock);
        idr_remove(idr, id);
-       spin_unlock_irq(&rhp->lock);
+       if (lock)
+               spin_unlock_irq(&rhp->lock);
+}
+
+static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
+{
+       _remove_handle(rhp, idr, id, 1);
+}
+
+static inline void remove_handle_nolock(struct c4iw_dev *rhp,
+                                        struct idr *idr, u32 id)
+{
+       _remove_handle(rhp, idr, id, 0);
 }
 
 struct c4iw_pd {
@@ -353,6 +420,8 @@ struct c4iw_qp_attributes {
        struct c4iw_ep *llp_stream_handle;
        u8 layer_etype;
        u8 ecode;
+       u16 sq_db_inc;
+       u16 rq_db_inc;
 };
 
 struct c4iw_qp {
@@ -427,6 +496,8 @@ static inline void insert_mmap(struct c4iw_ucontext *ucontext,
 
 enum c4iw_qp_attr_mask {
        C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
+       C4IW_QP_ATTR_SQ_DB = 1<<1,
+       C4IW_QP_ATTR_RQ_DB = 1<<2,
        C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
        C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
        C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
@@ -480,6 +551,23 @@ static inline int c4iw_convert_state(enum ib_qp_state ib_state)
        }
 }
 
+static inline int to_ib_qp_state(int c4iw_qp_state)
+{
+       switch (c4iw_qp_state) {
+       case C4IW_QP_STATE_IDLE:
+               return IB_QPS_INIT;
+       case C4IW_QP_STATE_RTS:
+               return IB_QPS_RTS;
+       case C4IW_QP_STATE_CLOSING:
+               return IB_QPS_SQD;
+       case C4IW_QP_STATE_TERMINATE:
+               return IB_QPS_SQE;
+       case C4IW_QP_STATE_ERROR:
+               return IB_QPS_ERR;
+       }
+       return IB_QPS_ERR;
+}
+
 static inline u32 c4iw_ib_to_tpt_access(int a)
 {
        return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
@@ -693,14 +781,20 @@ static inline int compute_wscale(int win)
        return wscale;
 }
 
+u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
+void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
+int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
+                       u32 reserved, u32 flags);
+void c4iw_id_table_free(struct c4iw_id_table *alloc);
+
 typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);
 
 int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
                     struct l2t_entry *l2t);
 void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
                   struct c4iw_dev_ucontext *uctx);
-u32 c4iw_get_resource(struct kfifo *fifo, spinlock_t *lock);
-void c4iw_put_resource(struct kfifo *fifo, u32 entry, spinlock_t *lock);
+u32 c4iw_get_resource(struct c4iw_id_table *id_table);
+void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
 int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
 int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
 int c4iw_pblpool_create(struct c4iw_rdev *rdev);
@@ -769,6 +863,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
                             struct ib_udata *udata);
 int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
                                 int attr_mask, struct ib_udata *udata);
+int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+                    int attr_mask, struct ib_qp_init_attr *init_attr);
 struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
 u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
 void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
@@ -797,5 +893,7 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
 extern struct cxgb4_client t4c_client;
 extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
 extern int c4iw_max_read_depth;
+extern int db_fc_threshold;
+
 
 #endif
index 40c8353..57e07c6 100644 (file)
@@ -131,10 +131,14 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
        stag_idx = (*stag) >> 8;
 
        if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) {
-               stag_idx = c4iw_get_resource(&rdev->resource.tpt_fifo,
-                                            &rdev->resource.tpt_fifo_lock);
+               stag_idx = c4iw_get_resource(&rdev->resource.tpt_table);
                if (!stag_idx)
                        return -ENOMEM;
+               mutex_lock(&rdev->stats.lock);
+               rdev->stats.stag.cur += 32;
+               if (rdev->stats.stag.cur > rdev->stats.stag.max)
+                       rdev->stats.stag.max = rdev->stats.stag.cur;
+               mutex_unlock(&rdev->stats.lock);
                *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
        }
        PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
@@ -165,9 +169,12 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
                                (rdev->lldi.vr->stag.start >> 5),
                                sizeof(tpt), &tpt);
 
-       if (reset_tpt_entry)
-               c4iw_put_resource(&rdev->resource.tpt_fifo, stag_idx,
-                                 &rdev->resource.tpt_fifo_lock);
+       if (reset_tpt_entry) {
+               c4iw_put_resource(&rdev->resource.tpt_table, stag_idx);
+               mutex_lock(&rdev->stats.lock);
+               rdev->stats.stag.cur -= 32;
+               mutex_unlock(&rdev->stats.lock);
+       }
        return err;
 }
 
@@ -686,8 +693,8 @@ int c4iw_dealloc_mw(struct ib_mw *mw)
        mhp = to_c4iw_mw(mw);
        rhp = mhp->rhp;
        mmid = (mw->rkey) >> 8;
-       deallocate_window(&rhp->rdev, mhp->attr.stag);
        remove_handle(rhp, &rhp->mmidr, mmid);
+       deallocate_window(&rhp->rdev, mhp->attr.stag);
        kfree(mhp);
        PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
        return 0;
@@ -789,12 +796,12 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr)
        mhp = to_c4iw_mr(ib_mr);
        rhp = mhp->rhp;
        mmid = mhp->attr.stag >> 8;
+       remove_handle(rhp, &rhp->mmidr, mmid);
        dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
                       mhp->attr.pbl_addr);
        if (mhp->attr.pbl_size)
                c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
                                  mhp->attr.pbl_size << 3);
-       remove_handle(rhp, &rhp->mmidr, mmid);
        if (mhp->kva)
                kfree((void *) (unsigned long) mhp->kva);
        if (mhp->umem)
index be1c18f..e084fdc 100644 (file)
@@ -188,8 +188,10 @@ static int c4iw_deallocate_pd(struct ib_pd *pd)
        php = to_c4iw_pd(pd);
        rhp = php->rhp;
        PDBG("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
-       c4iw_put_resource(&rhp->rdev.resource.pdid_fifo, php->pdid,
-                         &rhp->rdev.resource.pdid_fifo_lock);
+       c4iw_put_resource(&rhp->rdev.resource.pdid_table, php->pdid);
+       mutex_lock(&rhp->rdev.stats.lock);
+       rhp->rdev.stats.pd.cur--;
+       mutex_unlock(&rhp->rdev.stats.lock);
        kfree(php);
        return 0;
 }
@@ -204,14 +206,12 @@ static struct ib_pd *c4iw_allocate_pd(struct ib_device *ibdev,
 
        PDBG("%s ibdev %p\n", __func__, ibdev);
        rhp = (struct c4iw_dev *) ibdev;
-       pdid =  c4iw_get_resource(&rhp->rdev.resource.pdid_fifo,
-                                 &rhp->rdev.resource.pdid_fifo_lock);
+       pdid =  c4iw_get_resource(&rhp->rdev.resource.pdid_table);
        if (!pdid)
                return ERR_PTR(-EINVAL);
        php = kzalloc(sizeof(*php), GFP_KERNEL);
        if (!php) {
-               c4iw_put_resource(&rhp->rdev.resource.pdid_fifo, pdid,
-                                 &rhp->rdev.resource.pdid_fifo_lock);
+               c4iw_put_resource(&rhp->rdev.resource.pdid_table, pdid);
                return ERR_PTR(-ENOMEM);
        }
        php->pdid = pdid;
@@ -222,6 +222,11 @@ static struct ib_pd *c4iw_allocate_pd(struct ib_device *ibdev,
                        return ERR_PTR(-EFAULT);
                }
        }
+       mutex_lock(&rhp->rdev.stats.lock);
+       rhp->rdev.stats.pd.cur++;
+       if (rhp->rdev.stats.pd.cur > rhp->rdev.stats.pd.max)
+               rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur;
+       mutex_unlock(&rhp->rdev.stats.lock);
        PDBG("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
        return &php->ibpd;
 }
@@ -438,6 +443,7 @@ int c4iw_register_device(struct c4iw_dev *dev)
            (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
            (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
            (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+           (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
            (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
            (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
            (1ull << IB_USER_VERBS_CMD_POST_SEND) |
@@ -460,6 +466,7 @@ int c4iw_register_device(struct c4iw_dev *dev)
        dev->ibdev.destroy_ah = c4iw_ah_destroy;
        dev->ibdev.create_qp = c4iw_create_qp;
        dev->ibdev.modify_qp = c4iw_ib_modify_qp;
+       dev->ibdev.query_qp = c4iw_ib_query_qp;
        dev->ibdev.destroy_qp = c4iw_destroy_qp;
        dev->ibdev.create_cq = c4iw_create_cq;
        dev->ibdev.destroy_cq = c4iw_destroy_cq;
index 5f940ae..45aedf1 100644 (file)
 
 #include "iw_cxgb4.h"
 
+static int db_delay_usecs = 1;
+module_param(db_delay_usecs, int, 0644);
+MODULE_PARM_DESC(db_delay_usecs, "Usecs to delay awaiting db fifo to drain");
+
 static int ocqp_support = 1;
 module_param(ocqp_support, int, 0644);
 MODULE_PARM_DESC(ocqp_support, "Support on-chip SQs (default=1)");
 
+int db_fc_threshold = 2000;
+module_param(db_fc_threshold, int, 0644);
+MODULE_PARM_DESC(db_fc_threshold, "QP count/threshold that triggers automatic "
+                "db flow control mode (default = 2000)");
+
 static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
 {
        unsigned long flag;
@@ -1128,6 +1137,35 @@ out:
        return ret;
 }
 
+/*
+ * Called by the library when the qp has user dbs disabled due to
+ * a DB_FULL condition.  This function will single-thread all user
+ * DB rings to avoid overflowing the hw db-fifo.
+ */
+static int ring_kernel_db(struct c4iw_qp *qhp, u32 qid, u16 inc)
+{
+       int delay = db_delay_usecs;
+
+       mutex_lock(&qhp->rhp->db_mutex);
+       do {
+
+               /*
+                * The interrupt threshold is dbfifo_int_thresh << 6. So
+                * make sure we don't cross that and generate an interrupt.
+                */
+               if (cxgb4_dbfifo_count(qhp->rhp->rdev.lldi.ports[0], 1) <
+                   (qhp->rhp->rdev.lldi.dbfifo_int_thresh << 5)) {
+                       writel(V_QID(qid) | V_PIDX(inc), qhp->wq.db);
+                       break;
+               }
+               set_current_state(TASK_UNINTERRUPTIBLE);
+               schedule_timeout(usecs_to_jiffies(delay));
+               delay = min(delay << 1, 2000);
+       } while (1);
+       mutex_unlock(&qhp->rhp->db_mutex);
+       return 0;
+}
+
 int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
                   enum c4iw_qp_attr_mask mask,
                   struct c4iw_qp_attributes *attrs,
@@ -1176,6 +1214,15 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
                qhp->attr = newattr;
        }
 
+       if (mask & C4IW_QP_ATTR_SQ_DB) {
+               ret = ring_kernel_db(qhp, qhp->wq.sq.qid, attrs->sq_db_inc);
+               goto out;
+       }
+       if (mask & C4IW_QP_ATTR_RQ_DB) {
+               ret = ring_kernel_db(qhp, qhp->wq.rq.qid, attrs->rq_db_inc);
+               goto out;
+       }
+
        if (!(mask & C4IW_QP_ATTR_NEXT_STATE))
                goto out;
        if (qhp->attr.state == attrs->next_state)
@@ -1352,6 +1399,14 @@ out:
        return ret;
 }
 
+static int enable_qp_db(int id, void *p, void *data)
+{
+       struct c4iw_qp *qp = p;
+
+       t4_enable_wq_db(&qp->wq);
+       return 0;
+}
+
 int c4iw_destroy_qp(struct ib_qp *ib_qp)
 {
        struct c4iw_dev *rhp;
@@ -1369,7 +1424,16 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp)
                c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
        wait_event(qhp->wait, !qhp->ep);
 
-       remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
+       spin_lock_irq(&rhp->lock);
+       remove_handle_nolock(rhp, &rhp->qpidr, qhp->wq.sq.qid);
+       rhp->qpcnt--;
+       BUG_ON(rhp->qpcnt < 0);
+       if (rhp->qpcnt <= db_fc_threshold && rhp->db_state == FLOW_CONTROL) {
+               rhp->rdev.stats.db_state_transitions++;
+               rhp->db_state = NORMAL;
+               idr_for_each(&rhp->qpidr, enable_qp_db, NULL);
+       }
+       spin_unlock_irq(&rhp->lock);
        atomic_dec(&qhp->refcnt);
        wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 
@@ -1383,6 +1447,14 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp)
        return 0;
 }
 
+static int disable_qp_db(int id, void *p, void *data)
+{
+       struct c4iw_qp *qp = p;
+
+       t4_disable_wq_db(&qp->wq);
+       return 0;
+}
+
 struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
                             struct ib_udata *udata)
 {
@@ -1469,7 +1541,16 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
        init_waitqueue_head(&qhp->wait);
        atomic_set(&qhp->refcnt, 1);
 
-       ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
+       spin_lock_irq(&rhp->lock);
+       if (rhp->db_state != NORMAL)
+               t4_disable_wq_db(&qhp->wq);
+       if (++rhp->qpcnt > db_fc_threshold && rhp->db_state == NORMAL) {
+               rhp->rdev.stats.db_state_transitions++;
+               rhp->db_state = FLOW_CONTROL;
+               idr_for_each(&rhp->qpidr, disable_qp_db, NULL);
+       }
+       ret = insert_handle_nolock(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
+       spin_unlock_irq(&rhp->lock);
        if (ret)
                goto err2;
 
@@ -1613,6 +1694,15 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
                         C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
                         C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0;
 
+       /*
+        * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for
+        * ringing the queue db when we're in DB_FULL mode.
+        */
+       attrs.sq_db_inc = attr->sq_psn;
+       attrs.rq_db_inc = attr->rq_psn;
+       mask |= (attr_mask & IB_QP_SQ_PSN) ? C4IW_QP_ATTR_SQ_DB : 0;
+       mask |= (attr_mask & IB_QP_RQ_PSN) ? C4IW_QP_ATTR_RQ_DB : 0;
+
        return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0);
 }
 
@@ -1621,3 +1711,14 @@ struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn)
        PDBG("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
        return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn);
 }
+
+int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+                    int attr_mask, struct ib_qp_init_attr *init_attr)
+{
+       struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
+
+       memset(attr, 0, sizeof *attr);
+       memset(init_attr, 0, sizeof *init_attr);
+       attr->qp_state = to_ib_qp_state(qhp->attr.state);
+       return 0;
+}
index 407ff39..cdef4d7 100644 (file)
  * SOFTWARE.
  */
 /* Crude resource management */
-#include <linux/kernel.h>
-#include <linux/random.h>
-#include <linux/slab.h>
-#include <linux/kfifo.h>
 #include <linux/spinlock.h>
-#include <linux/errno.h>
 #include <linux/genalloc.h>
 #include <linux/ratelimit.h>
 #include "iw_cxgb4.h"
 
-#define RANDOM_SIZE 16
-
-static int __c4iw_init_resource_fifo(struct kfifo *fifo,
-                                  spinlock_t *fifo_lock,
-                                  u32 nr, u32 skip_low,
-                                  u32 skip_high,
-                                  int random)
-{
-       u32 i, j, entry = 0, idx;
-       u32 random_bytes;
-       u32 rarray[16];
-       spin_lock_init(fifo_lock);
-
-       if (kfifo_alloc(fifo, nr * sizeof(u32), GFP_KERNEL))
-               return -ENOMEM;
-
-       for (i = 0; i < skip_low + skip_high; i++)
-               kfifo_in(fifo, (unsigned char *) &entry, sizeof(u32));
-       if (random) {
-               j = 0;
-               random_bytes = random32();
-               for (i = 0; i < RANDOM_SIZE; i++)
-                       rarray[i] = i + skip_low;
-               for (i = skip_low + RANDOM_SIZE; i < nr - skip_high; i++) {
-                       if (j >= RANDOM_SIZE) {
-                               j = 0;
-                               random_bytes = random32();
-                       }
-                       idx = (random_bytes >> (j * 2)) & 0xF;
-                       kfifo_in(fifo,
-                               (unsigned char *) &rarray[idx],
-                               sizeof(u32));
-                       rarray[idx] = i;
-                       j++;
-               }
-               for (i = 0; i < RANDOM_SIZE; i++)
-                       kfifo_in(fifo,
-                               (unsigned char *) &rarray[i],
-                               sizeof(u32));
-       } else
-               for (i = skip_low; i < nr - skip_high; i++)
-                       kfifo_in(fifo, (unsigned char *) &i, sizeof(u32));
-
-       for (i = 0; i < skip_low + skip_high; i++)
-               if (kfifo_out_locked(fifo, (unsigned char *) &entry,
-                                    sizeof(u32), fifo_lock))
-                       break;
-       return 0;
-}
-
-static int c4iw_init_resource_fifo(struct kfifo *fifo, spinlock_t * fifo_lock,
-                                  u32 nr, u32 skip_low, u32 skip_high)
-{
-       return __c4iw_init_resource_fifo(fifo, fifo_lock, nr, skip_low,
-                                         skip_high, 0);
-}
-
-static int c4iw_init_resource_fifo_random(struct kfifo *fifo,
-                                  spinlock_t *fifo_lock,
-                                  u32 nr, u32 skip_low, u32 skip_high)
-{
-       return __c4iw_init_resource_fifo(fifo, fifo_lock, nr, skip_low,
-                                         skip_high, 1);
-}
-
-static int c4iw_init_qid_fifo(struct c4iw_rdev *rdev)
+static int c4iw_init_qid_table(struct c4iw_rdev *rdev)
 {
        u32 i;
 
-       spin_lock_init(&rdev->resource.qid_fifo_lock);
-
-       if (kfifo_alloc(&rdev->resource.qid_fifo, rdev->lldi.vr->qp.size *
-                       sizeof(u32), GFP_KERNEL))
+       if (c4iw_id_table_alloc(&rdev->resource.qid_table,
+                               rdev->lldi.vr->qp.start,
+                               rdev->lldi.vr->qp.size,
+                               rdev->lldi.vr->qp.size, 0))
                return -ENOMEM;
 
        for (i = rdev->lldi.vr->qp.start;
-            i < rdev->lldi.vr->qp.start + rdev->lldi.vr->qp.size; i++)
+               i < rdev->lldi.vr->qp.start + rdev->lldi.vr->qp.size; i++)
                if (!(i & rdev->qpmask))
-                       kfifo_in(&rdev->resource.qid_fifo,
-                                   (unsigned char *) &i, sizeof(u32));
+                       c4iw_id_free(&rdev->resource.qid_table, i);
        return 0;
 }
 
@@ -127,44 +56,42 @@ static int c4iw_init_qid_fifo(struct c4iw_rdev *rdev)
 int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid)
 {
        int err = 0;
-       err = c4iw_init_resource_fifo_random(&rdev->resource.tpt_fifo,
-                                            &rdev->resource.tpt_fifo_lock,
-                                            nr_tpt, 1, 0);
+       err = c4iw_id_table_alloc(&rdev->resource.tpt_table, 0, nr_tpt, 1,
+                                       C4IW_ID_TABLE_F_RANDOM);
        if (err)
                goto tpt_err;
-       err = c4iw_init_qid_fifo(rdev);
+       err = c4iw_init_qid_table(rdev);
        if (err)
                goto qid_err;
-       err = c4iw_init_resource_fifo(&rdev->resource.pdid_fifo,
-                                     &rdev->resource.pdid_fifo_lock,
-                                     nr_pdid, 1, 0);
+       err = c4iw_id_table_alloc(&rdev->resource.pdid_table, 0,
+                                       nr_pdid, 1, 0);
        if (err)
                goto pdid_err;
        return 0;
-pdid_err:
-       kfifo_free(&rdev->resource.qid_fifo);
-qid_err:
-       kfifo_free(&rdev->resource.tpt_fifo);
-tpt_err:
+ pdid_err:
+       c4iw_id_table_free(&rdev->resource.qid_table);
+ qid_err:
+       c4iw_id_table_free(&rdev->resource.tpt_table);
+ tpt_err:
        return -ENOMEM;
 }
 
 /*
  * returns 0 if no resource available
  */
-u32 c4iw_get_resource(struct kfifo *fifo, spinlock_t *lock)
+u32 c4iw_get_resource(struct c4iw_id_table *id_table)
 {
        u32 entry;
-       if (kfifo_out_locked(fifo, (unsigned char *) &entry, sizeof(u32), lock))
-               return entry;
-       else
+       entry = c4iw_id_alloc(id_table);
+       if (entry == (u32)(-1))
                return 0;
+       return entry;
 }
 
-void c4iw_put_resource(struct kfifo *fifo, u32 entry, spinlock_t *lock)
+void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry)
 {
        PDBG("%s entry 0x%x\n", __func__, entry);
-       kfifo_in_locked(fifo, (unsigned char *) &entry, sizeof(u32), lock);
+       c4iw_id_free(id_table, entry);
 }
 
 u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
@@ -181,10 +108,12 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
                qid = entry->qid;
                kfree(entry);
        } else {
-               qid = c4iw_get_resource(&rdev->resource.qid_fifo,
-                                       &rdev->resource.qid_fifo_lock);
+               qid = c4iw_get_resource(&rdev->resource.qid_table);
                if (!qid)
                        goto out;
+               mutex_lock(&rdev->stats.lock);
+               rdev->stats.qid.cur += rdev->qpmask + 1;
+               mutex_unlock(&rdev->stats.lock);
                for (i = qid+1; i & rdev->qpmask; i++) {
                        entry = kmalloc(sizeof *entry, GFP_KERNEL);
                        if (!entry)
@@ -213,6 +142,10 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
 out:
        mutex_unlock(&uctx->lock);
        PDBG("%s qid 0x%x\n", __func__, qid);
+       mutex_lock(&rdev->stats.lock);
+       if (rdev->stats.qid.cur > rdev->stats.qid.max)
+               rdev->stats.qid.max = rdev->stats.qid.cur;
+       mutex_unlock(&rdev->stats.lock);
        return qid;
 }
 
@@ -245,10 +178,12 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
                qid = entry->qid;
                kfree(entry);
        } else {
-               qid = c4iw_get_resource(&rdev->resource.qid_fifo,
-                                       &rdev->resource.qid_fifo_lock);
+               qid = c4iw_get_resource(&rdev->resource.qid_table);
                if (!qid)
                        goto out;
+               mutex_lock(&rdev->stats.lock);
+               rdev->stats.qid.cur += rdev->qpmask + 1;
+               mutex_unlock(&rdev->stats.lock);
                for (i = qid+1; i & rdev->qpmask; i++) {
                        entry = kmalloc(sizeof *entry, GFP_KERNEL);
                        if (!entry)
@@ -277,6 +212,10 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
 out:
        mutex_unlock(&uctx->lock);
        PDBG("%s qid 0x%x\n", __func__, qid);
+       mutex_lock(&rdev->stats.lock);
+       if (rdev->stats.qid.cur > rdev->stats.qid.max)
+               rdev->stats.qid.max = rdev->stats.qid.cur;
+       mutex_unlock(&rdev->stats.lock);
        return qid;
 }
 
@@ -297,9 +236,9 @@ void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
 
 void c4iw_destroy_resource(struct c4iw_resource *rscp)
 {
-       kfifo_free(&rscp->tpt_fifo);
-       kfifo_free(&rscp->qid_fifo);
-       kfifo_free(&rscp->pdid_fifo);
+       c4iw_id_table_free(&rscp->tpt_table);
+       c4iw_id_table_free(&rscp->qid_table);
+       c4iw_id_table_free(&rscp->pdid_table);
 }
 
 /*
@@ -312,15 +251,23 @@ u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size)
 {
        unsigned long addr = gen_pool_alloc(rdev->pbl_pool, size);
        PDBG("%s addr 0x%x size %d\n", __func__, (u32)addr, size);
-       if (!addr)
-               printk_ratelimited(KERN_WARNING MOD "%s: Out of PBL memory\n",
-                      pci_name(rdev->lldi.pdev));
+       mutex_lock(&rdev->stats.lock);
+       if (addr) {
+               rdev->stats.pbl.cur += roundup(size, 1 << MIN_PBL_SHIFT);
+               if (rdev->stats.pbl.cur > rdev->stats.pbl.max)
+                       rdev->stats.pbl.max = rdev->stats.pbl.cur;
+       } else
+               rdev->stats.pbl.fail++;
+       mutex_unlock(&rdev->stats.lock);
        return (u32)addr;
 }
 
 void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size)
 {
        PDBG("%s addr 0x%x size %d\n", __func__, addr, size);
+       mutex_lock(&rdev->stats.lock);
+       rdev->stats.pbl.cur -= roundup(size, 1 << MIN_PBL_SHIFT);
+       mutex_unlock(&rdev->stats.lock);
        gen_pool_free(rdev->pbl_pool, (unsigned long)addr, size);
 }
 
@@ -377,12 +324,23 @@ u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size)
        if (!addr)
                printk_ratelimited(KERN_WARNING MOD "%s: Out of RQT memory\n",
                       pci_name(rdev->lldi.pdev));
+       mutex_lock(&rdev->stats.lock);
+       if (addr) {
+               rdev->stats.rqt.cur += roundup(size << 6, 1 << MIN_RQT_SHIFT);
+               if (rdev->stats.rqt.cur > rdev->stats.rqt.max)
+                       rdev->stats.rqt.max = rdev->stats.rqt.cur;
+       } else
+               rdev->stats.rqt.fail++;
+       mutex_unlock(&rdev->stats.lock);
        return (u32)addr;
 }
 
 void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size)
 {
        PDBG("%s addr 0x%x size %d\n", __func__, addr, size << 6);
+       mutex_lock(&rdev->stats.lock);
+       rdev->stats.rqt.cur -= roundup(size << 6, 1 << MIN_RQT_SHIFT);
+       mutex_unlock(&rdev->stats.lock);
        gen_pool_free(rdev->rqt_pool, (unsigned long)addr, size << 6);
 }
 
@@ -433,12 +391,22 @@ u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size)
 {
        unsigned long addr = gen_pool_alloc(rdev->ocqp_pool, size);
        PDBG("%s addr 0x%x size %d\n", __func__, (u32)addr, size);
+       if (addr) {
+               mutex_lock(&rdev->stats.lock);
+               rdev->stats.ocqp.cur += roundup(size, 1 << MIN_OCQP_SHIFT);
+               if (rdev->stats.ocqp.cur > rdev->stats.ocqp.max)
+                       rdev->stats.ocqp.max = rdev->stats.ocqp.cur;
+               mutex_unlock(&rdev->stats.lock);
+       }
        return (u32)addr;
 }
 
 void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size)
 {
        PDBG("%s addr 0x%x size %d\n", __func__, addr, size);
+       mutex_lock(&rdev->stats.lock);
+       rdev->stats.ocqp.cur -= roundup(size, 1 << MIN_OCQP_SHIFT);
+       mutex_unlock(&rdev->stats.lock);
        gen_pool_free(rdev->ocqp_pool, (unsigned long)addr, size);
 }
 
index c0221ee..16f26ab 100644 (file)
@@ -62,6 +62,10 @@ struct t4_status_page {
        __be16 pidx;
        u8 qp_err;      /* flit 1 - sw owns */
        u8 db_off;
+       u8 pad;
+       u16 host_wq_pidx;
+       u16 host_cidx;
+       u16 host_pidx;
 };
 
 #define T4_EQ_ENTRY_SIZE 64
@@ -375,6 +379,16 @@ static inline void t4_rq_consume(struct t4_wq *wq)
                wq->rq.cidx = 0;
 }
 
+static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)
+{
+       return wq->rq.queue[wq->rq.size].status.host_wq_pidx;
+}
+
+static inline u16 t4_rq_wq_size(struct t4_wq *wq)
+{
+               return wq->rq.size * T4_RQ_NUM_SLOTS;
+}
+
 static inline int t4_sq_onchip(struct t4_sq *sq)
 {
        return sq->flags & T4_SQ_ONCHIP;
@@ -412,6 +426,16 @@ static inline void t4_sq_consume(struct t4_wq *wq)
                wq->sq.cidx = 0;
 }
 
+static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)
+{
+       return wq->sq.queue[wq->sq.size].status.host_wq_pidx;
+}
+
+static inline u16 t4_sq_wq_size(struct t4_wq *wq)
+{
+               return wq->sq.size * T4_SQ_NUM_SLOTS;
+}
+
 static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc)
 {
        wmb();
index e6669d5..32b754c 100644 (file)
@@ -32,7 +32,7 @@
 #ifndef __C4IW_USER_H__
 #define __C4IW_USER_H__
 
-#define C4IW_UVERBS_ABI_VERSION        1
+#define C4IW_UVERBS_ABI_VERSION        2
 
 /*
  * Make sure that all structs defined in this file remain laid out so
index 1d7aea1..7cc3054 100644 (file)
@@ -596,8 +596,7 @@ static void ipath_ht_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 
        ipath_format_hwerrors(hwerrs,
                              ipath_6110_hwerror_msgs,
-                             sizeof(ipath_6110_hwerror_msgs) /
-                             sizeof(ipath_6110_hwerror_msgs[0]),
+                             ARRAY_SIZE(ipath_6110_hwerror_msgs),
                              msg, msgl);
 
        if (hwerrs & (_IPATH_HTLINK0_CRCBITS | _IPATH_HTLINK1_CRCBITS))
index c0a03ac..26dfbc8 100644 (file)
@@ -209,8 +209,7 @@ void ipath_format_hwerrors(u64 hwerrs,
 {
        int i;
        const int glen =
-           sizeof(ipath_generic_hwerror_msgs) /
-           sizeof(ipath_generic_hwerror_msgs[0]);
+           ARRAY_SIZE(ipath_generic_hwerror_msgs);
 
        for (i=0; i<glen; i++) {
                if (hwerrs & ipath_generic_hwerror_msgs[i].mask) {
index 77c8cb4..6d4ef71 100644 (file)
@@ -50,7 +50,7 @@ static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)
        struct ib_cq *ibcq;
 
        if (type != MLX4_EVENT_TYPE_CQ_ERROR) {
-               printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
+               pr_warn("Unexpected event type %d "
                       "on CQ %06x\n", type, cq->cqn);
                return;
        }
@@ -222,6 +222,9 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
                uar = &dev->priv_uar;
        }
 
+       if (dev->eq_table)
+               vector = dev->eq_table[vector % ibdev->num_comp_vectors];
+
        err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
                            cq->db.dma, &cq->mcq, vector, 0);
        if (err)
@@ -463,7 +466,7 @@ static void dump_cqe(void *cqe)
 {
        __be32 *buf = cqe;
 
-       printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
+       pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
               be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),
               be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),
               be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));
@@ -473,7 +476,7 @@ static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,
                                     struct ib_wc *wc)
 {
        if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {
-               printk(KERN_DEBUG "local QP operation err "
+               pr_debug("local QP operation err "
                       "(QPN %06x, WQE index %x, vendor syndrome %02x, "
                       "opcode = %02x)\n",
                       be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),
@@ -576,7 +579,7 @@ repoll:
 
        if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_OPCODE_NOP &&
                     is_send)) {
-               printk(KERN_WARNING "Completion for NOP opcode detected!\n");
+               pr_warn("Completion for NOP opcode detected!\n");
                return -EINVAL;
        }
 
@@ -606,7 +609,7 @@ repoll:
                mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,
                                       be32_to_cpu(cqe->vlan_my_qpn));
                if (unlikely(!mqp)) {
-                       printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n",
+                       pr_warn("CQ %06x with entry for unknown QPN %06x\n",
                               cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK);
                        return -EINVAL;
                }
index b948b6d..ee1c577 100644 (file)
@@ -789,7 +789,7 @@ static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
                list_del(&ge->list);
                kfree(ge);
        } else
-               printk(KERN_WARNING "could not find mgid entry\n");
+               pr_warn("could not find mgid entry\n");
 
        mutex_unlock(&mqp->mutex);
 
@@ -902,7 +902,7 @@ static void update_gids_task(struct work_struct *work)
 
        mailbox = mlx4_alloc_cmd_mailbox(dev);
        if (IS_ERR(mailbox)) {
-               printk(KERN_WARNING "update gid table failed %ld\n", PTR_ERR(mailbox));
+               pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox));
                return;
        }
 
@@ -913,7 +913,7 @@ static void update_gids_task(struct work_struct *work)
                       1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
                       MLX4_CMD_NATIVE);
        if (err)
-               printk(KERN_WARNING "set port command failed\n");
+               pr_warn("set port command failed\n");
        else {
                memcpy(gw->dev->iboe.gid_table[gw->port - 1], gw->gids, sizeof gw->gids);
                event.device = &gw->dev->ib_dev;
@@ -1076,18 +1076,98 @@ static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event
        return NOTIFY_DONE;
 }
 
+static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
+{
+       char name[32];
+       int eq_per_port = 0;
+       int added_eqs = 0;
+       int total_eqs = 0;
+       int i, j, eq;
+
+       /* Init eq table */
+       ibdev->eq_table = NULL;
+       ibdev->eq_added = 0;
+
+       /* Legacy mode? */
+       if (dev->caps.comp_pool == 0)
+               return;
+
+       eq_per_port = rounddown_pow_of_two(dev->caps.comp_pool/
+                                       dev->caps.num_ports);
+
+       /* Init eq table */
+       added_eqs = 0;
+       mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
+               added_eqs += eq_per_port;
+
+       total_eqs = dev->caps.num_comp_vectors + added_eqs;
+
+       ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL);
+       if (!ibdev->eq_table)
+               return;
+
+       ibdev->eq_added = added_eqs;
+
+       eq = 0;
+       mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) {
+               for (j = 0; j < eq_per_port; j++) {
+                       sprintf(name, "mlx4-ib-%d-%d@%s",
+                               i, j, dev->pdev->bus->name);
+                       /* Set IRQ for specific name (per ring) */
+                       if (mlx4_assign_eq(dev, name, &ibdev->eq_table[eq])) {
+                               /* Use legacy (same as mlx4_en driver) */
+                               pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq);
+                               ibdev->eq_table[eq] =
+                                       (eq % dev->caps.num_comp_vectors);
+                       }
+                       eq++;
+               }
+       }
+
+       /* Fill the reset of the vector with legacy EQ */
+       for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++)
+               ibdev->eq_table[eq++] = i;
+
+       /* Advertise the new number of EQs to clients */
+       ibdev->ib_dev.num_comp_vectors = total_eqs;
+}
+
+static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
+{
+       int i;
+       int total_eqs;
+
+       /* Reset the advertised EQ number */
+       ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
+
+       /* Free only the added eqs */
+       for (i = 0; i < ibdev->eq_added; i++) {
+               /* Don't free legacy eqs if used */
+               if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors)
+                       continue;
+               mlx4_release_eq(dev, ibdev->eq_table[i]);
+       }
+
+       total_eqs = dev->caps.num_comp_vectors + ibdev->eq_added;
+       memset(ibdev->eq_table, 0, total_eqs * sizeof(int));
+       kfree(ibdev->eq_table);
+
+       ibdev->eq_table = NULL;
+       ibdev->eq_added = 0;
+}
+
 static void *mlx4_ib_add(struct mlx4_dev *dev)
 {
        struct mlx4_ib_dev *ibdev;
        int num_ports = 0;
-       int i;
+       int i, j;
        int err;
        struct mlx4_ib_iboe *iboe;
 
-       printk_once(KERN_INFO "%s", mlx4_ib_version);
+       pr_info_once("%s", mlx4_ib_version);
 
        if (mlx4_is_mfunc(dev)) {
-               printk(KERN_WARNING "IB not yet supported in SRIOV\n");
+               pr_warn("IB not yet supported in SRIOV\n");
                return NULL;
        }
 
@@ -1210,6 +1290,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
                        (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
        }
 
+       mlx4_ib_alloc_eqs(dev, ibdev);
+
        spin_lock_init(&iboe->lock);
 
        if (init_node_data(ibdev))
@@ -1241,9 +1323,9 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
                        goto err_reg;
        }
 
-       for (i = 0; i < ARRAY_SIZE(mlx4_class_attributes); ++i) {
+       for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) {
                if (device_create_file(&ibdev->ib_dev.dev,
-                                      mlx4_class_attributes[i]))
+                                      mlx4_class_attributes[j]))
                        goto err_notif;
        }
 
@@ -1253,7 +1335,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 
 err_notif:
        if (unregister_netdevice_notifier(&ibdev->iboe.nb))
-               printk(KERN_WARNING "failure unregistering notifier\n");
+               pr_warn("failure unregistering notifier\n");
        flush_workqueue(wq);
 
 err_reg:
@@ -1288,7 +1370,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
        ib_unregister_device(&ibdev->ib_dev);
        if (ibdev->iboe.nb.notifier_call) {
                if (unregister_netdevice_notifier(&ibdev->iboe.nb))
-                       printk(KERN_WARNING "failure unregistering notifier\n");
+                       pr_warn("failure unregistering notifier\n");
                ibdev->iboe.nb.notifier_call = NULL;
        }
        iounmap(ibdev->uar_map);
@@ -1298,6 +1380,8 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
        mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB)
                mlx4_CLOSE_PORT(dev, p);
 
+       mlx4_ib_free_eqs(dev, ibdev);
+
        mlx4_uar_free(dev, &ibdev->priv_uar);
        mlx4_pd_free(dev, ibdev->priv_pdn);
        ib_dealloc_device(&ibdev->ib_dev);
index ed80345..e62297c 100644 (file)
@@ -202,6 +202,8 @@ struct mlx4_ib_dev {
        bool                    ib_active;
        struct mlx4_ib_iboe     iboe;
        int                     counters[MLX4_MAX_PORTS];
+       int                    *eq_table;
+       int                     eq_added;
 };
 
 static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev)
index dca55b1..bbaf617 100644 (file)
@@ -338,7 +338,7 @@ int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
 
        err = mlx4_SYNC_TPT(mdev);
        if (err)
-               printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when "
+               pr_warn("SYNC_TPT error %d when "
                       "unmapping FMRs\n", err);
 
        return 0;
index 3a78489..ceb3332 100644 (file)
@@ -84,6 +84,11 @@ enum {
        MLX4_IB_CACHE_LINE_SIZE = 64,
 };
 
+enum {
+       MLX4_RAW_QP_MTU         = 7,
+       MLX4_RAW_QP_MSGMAX      = 31,
+};
+
 static const __be32 mlx4_ib_opcode[] = {
        [IB_WR_SEND]                            = cpu_to_be32(MLX4_OPCODE_SEND),
        [IB_WR_LSO]                             = cpu_to_be32(MLX4_OPCODE_LSO),
@@ -256,7 +261,7 @@ static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
                        event.event = IB_EVENT_QP_ACCESS_ERR;
                        break;
                default:
-                       printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
+                       pr_warn("Unexpected event type %d "
                               "on QP %06x\n", type, qp->qpn);
                        return;
                }
@@ -573,7 +578,12 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
        if (sqpn) {
                qpn = sqpn;
        } else {
-               err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
+               /* Raw packet QPNs must be aligned to 8 bits. If not, the WQE
+                * BlueFlame setup flow wrongly causes VLAN insertion. */
+               if (init_attr->qp_type == IB_QPT_RAW_PACKET)
+                       err = mlx4_qp_reserve_range(dev->dev, 1, 1 << 8, &qpn);
+               else
+                       err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
                if (err)
                        goto err_wrid;
        }
@@ -715,7 +725,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
        if (qp->state != IB_QPS_RESET)
                if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
                                   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
-                       printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",
+                       pr_warn("modify QP %06x to RESET failed.\n",
                               qp->mqp.qpn);
 
        get_cqs(qp, &send_cq, &recv_cq);
@@ -791,6 +801,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
        case IB_QPT_RC:
        case IB_QPT_UC:
        case IB_QPT_UD:
+       case IB_QPT_RAW_PACKET:
        {
                qp = kzalloc(sizeof *qp, GFP_KERNEL);
                if (!qp)
@@ -872,7 +883,8 @@ static int to_mlx4_st(enum ib_qp_type type)
        case IB_QPT_XRC_INI:
        case IB_QPT_XRC_TGT:    return MLX4_QP_ST_XRC;
        case IB_QPT_SMI:
-       case IB_QPT_GSI:        return MLX4_QP_ST_MLX;
+       case IB_QPT_GSI:
+       case IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX;
        default:                return -1;
        }
 }
@@ -946,7 +958,7 @@ static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
 
        if (ah->ah_flags & IB_AH_GRH) {
                if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
-                       printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
+                       pr_err("sgid_index (%u) too large. max is %d\n",
                               ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
                        return -1;
                }
@@ -1042,6 +1054,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 
        if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
                context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
+       else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
+               context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
        else if (ibqp->qp_type == IB_QPT_UD) {
                if (qp->flags & MLX4_IB_QP_LSO)
                        context->mtu_msgmax = (IB_MTU_4096 << 5) |
@@ -1050,7 +1064,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
                        context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
        } else if (attr_mask & IB_QP_PATH_MTU) {
                if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
-                       printk(KERN_ERR "path MTU (%u) is invalid\n",
+                       pr_err("path MTU (%u) is invalid\n",
                               attr->path_mtu);
                        goto out;
                }
@@ -1200,7 +1214,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
        if (cur_state == IB_QPS_INIT &&
            new_state == IB_QPS_RTR  &&
            (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
-            ibqp->qp_type == IB_QPT_UD)) {
+            ibqp->qp_type == IB_QPT_UD ||
+            ibqp->qp_type == IB_QPT_RAW_PACKET)) {
                context->pri_path.sched_queue = (qp->port - 1) << 6;
                if (is_qp0(dev, qp))
                        context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
@@ -1266,7 +1281,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
        if (is_qp0(dev, qp)) {
                if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
                        if (mlx4_INIT_PORT(dev->dev, qp->port))
-                               printk(KERN_WARNING "INIT_PORT failed for port %d\n",
+                               pr_warn("INIT_PORT failed for port %d\n",
                                       qp->port);
 
                if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
@@ -1319,6 +1334,11 @@ int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
                goto out;
        }
 
+       if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
+           (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
+            IB_LINK_LAYER_ETHERNET))
+               goto out;
+
        if (attr_mask & IB_QP_PKEY_INDEX) {
                int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
                if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
@@ -1424,6 +1444,9 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
 
        if (is_eth) {
                u8 *smac;
+               u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
+
+               mlx->sched_prio = cpu_to_be16(pcp);
 
                memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
                /* FIXME: cache smac value? */
@@ -1434,10 +1457,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
                if (!is_vlan) {
                        sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
                } else {
-                       u16 pcp;
-
                        sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
-                       pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
                        sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
                }
        } else {
@@ -1460,16 +1480,16 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
        header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
 
        if (0) {
-               printk(KERN_ERR "built UD header of size %d:\n", header_size);
+               pr_err("built UD header of size %d:\n", header_size);
                for (i = 0; i < header_size / 4; ++i) {
                        if (i % 8 == 0)
-                               printk("  [%02x] ", i * 4);
-                       printk(" %08x",
-                              be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
+                               pr_err("  [%02x] ", i * 4);
+                       pr_cont(" %08x",
+                               be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
                        if ((i + 1) % 8 == 0)
-                               printk("\n");
+                               pr_cont("\n");
                }
-               printk("\n");
+               pr_err("\n");
        }
 
        /*
index 39542f3..60c5fb0 100644 (file)
@@ -59,7 +59,7 @@ static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
                        event.event = IB_EVENT_SRQ_ERR;
                        break;
                default:
-                       printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
+                       pr_warn("Unexpected event type %d "
                               "on SRQ %06x\n", type, srq->srqn);
                        return;
                }
diff --git a/drivers/infiniband/hw/ocrdma/Kconfig b/drivers/infiniband/hw/ocrdma/Kconfig
new file mode 100644 (file)
index 0000000..b5b6056
--- /dev/null
@@ -0,0 +1,8 @@
+config INFINIBAND_OCRDMA
+       tristate "Emulex One Connect HCA support"
+       depends on ETHERNET && NETDEVICES && PCI && (IPV6 || IPV6=n)
+       select NET_VENDOR_EMULEX
+       select BE2NET
+       ---help---
+         This driver provides low-level InfiniBand over Ethernet
+         support for Emulex One Connect host channel adapters (HCAs).
diff --git a/drivers/infiniband/hw/ocrdma/Makefile b/drivers/infiniband/hw/ocrdma/Makefile
new file mode 100644 (file)
index 0000000..06a5bed
--- /dev/null
@@ -0,0 +1,5 @@
+ccflags-y := -Idrivers/net/ethernet/emulex/benet
+
+obj-$(CONFIG_INFINIBAND_OCRDMA)        += ocrdma.o
+
+ocrdma-y :=    ocrdma_main.o ocrdma_verbs.o ocrdma_hw.o ocrdma_ah.o
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h b/drivers/infiniband/hw/ocrdma/ocrdma.h
new file mode 100644 (file)
index 0000000..85a69c9
--- /dev/null
@@ -0,0 +1,393 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#ifndef __OCRDMA_H__
+#define __OCRDMA_H__
+
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/pci.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#include <be_roce.h>
+#include "ocrdma_sli.h"
+
+#define OCRDMA_ROCE_DEV_VERSION "1.0.0"
+#define OCRDMA_NODE_DESC "Emulex OneConnect RoCE HCA"
+
+#define ocrdma_err(format, arg...) printk(KERN_ERR format, ##arg)
+
+#define OCRDMA_MAX_AH 512
+
+#define OCRDMA_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
+
+struct ocrdma_dev_attr {
+       u8 fw_ver[32];
+       u32 vendor_id;
+       u32 device_id;
+       u16 max_pd;
+       u16 max_cq;
+       u16 max_cqe;
+       u16 max_qp;
+       u16 max_wqe;
+       u16 max_rqe;
+       u32 max_inline_data;
+       int max_send_sge;
+       int max_recv_sge;
+       int max_mr;
+       u64 max_mr_size;
+       u32 max_num_mr_pbl;
+       int max_fmr;
+       int max_map_per_fmr;
+       int max_pages_per_frmr;
+       u16 max_ord_per_qp;
+       u16 max_ird_per_qp;
+
+       int device_cap_flags;
+       u8 cq_overflow_detect;
+       u8 srq_supported;
+
+       u32 wqe_size;
+       u32 rqe_size;
+       u32 ird_page_size;
+       u8 local_ca_ack_delay;
+       u8 ird;
+       u8 num_ird_pages;
+};
+
+struct ocrdma_pbl {
+       void *va;
+       dma_addr_t pa;
+};
+
+struct ocrdma_queue_info {
+       void *va;
+       dma_addr_t dma;
+       u32 size;
+       u16 len;
+       u16 entry_size;         /* Size of an element in the queue */
+       u16 id;                 /* qid, where to ring the doorbell. */
+       u16 head, tail;
+       bool created;
+       atomic_t used;          /* Number of valid elements in the queue */
+};
+
+struct ocrdma_eq {
+       struct ocrdma_queue_info q;
+       u32 vector;
+       int cq_cnt;
+       struct ocrdma_dev *dev;
+       char irq_name[32];
+};
+
+struct ocrdma_mq {
+       struct ocrdma_queue_info sq;
+       struct ocrdma_queue_info cq;
+       bool rearm_cq;
+};
+
+struct mqe_ctx {
+       struct mutex lock; /* for serializing mailbox commands on MQ */
+       wait_queue_head_t cmd_wait;
+       u32 tag;
+       u16 cqe_status;
+       u16 ext_status;
+       bool cmd_done;
+};
+
+struct ocrdma_dev {
+       struct ib_device ibdev;
+       struct ocrdma_dev_attr attr;
+
+       struct mutex dev_lock; /* provides syncronise access to device data */
+       spinlock_t flush_q_lock ____cacheline_aligned;
+
+       struct ocrdma_cq **cq_tbl;
+       struct ocrdma_qp **qp_tbl;
+
+       struct ocrdma_eq meq;
+       struct ocrdma_eq *qp_eq_tbl;
+       int eq_cnt;
+       u16 base_eqid;
+       u16 max_eq;
+
+       union ib_gid *sgid_tbl;
+       /* provided synchronization to sgid table for
+        * updating gid entries triggered by notifier.
+        */
+       spinlock_t sgid_lock;
+
+       int gsi_qp_created;
+       struct ocrdma_cq *gsi_sqcq;
+       struct ocrdma_cq *gsi_rqcq;
+
+       struct {
+               struct ocrdma_av *va;
+               dma_addr_t pa;
+               u32 size;
+               u32 num_ah;
+               /* provide synchronization for av
+                * entry allocations.
+                */
+               spinlock_t lock;
+               u32 ahid;
+               struct ocrdma_pbl pbl;
+       } av_tbl;
+
+       void *mbx_cmd;
+       struct ocrdma_mq mq;
+       struct mqe_ctx mqe_ctx;
+
+       struct be_dev_info nic_info;
+
+       struct list_head entry;
+       struct rcu_head rcu;
+       int id;
+};
+
+struct ocrdma_cq {
+       struct ib_cq ibcq;
+       struct ocrdma_dev *dev;
+       struct ocrdma_cqe *va;
+       u32 phase;
+       u32 getp;       /* pointer to pending wrs to
+                        * return to stack, wrap arounds
+                        * at max_hw_cqe
+                        */
+       u32 max_hw_cqe;
+       bool phase_change;
+       bool armed, solicited;
+       bool arm_needed;
+
+       spinlock_t cq_lock ____cacheline_aligned; /* provide synchronization
+                                                  * to cq polling
+                                                  */
+       /* syncronizes cq completion handler invoked from multiple context */
+       spinlock_t comp_handler_lock ____cacheline_aligned;
+       u16 id;
+       u16 eqn;
+
+       struct ocrdma_ucontext *ucontext;
+       dma_addr_t pa;
+       u32 len;
+       atomic_t use_cnt;
+
+       /* head of all qp's sq and rq for which cqes need to be flushed
+        * by the software.
+        */
+       struct list_head sq_head, rq_head;
+};
+
+struct ocrdma_pd {
+       struct ib_pd ibpd;
+       struct ocrdma_dev *dev;
+       struct ocrdma_ucontext *uctx;
+       atomic_t use_cnt;
+       u32 id;
+       int num_dpp_qp;
+       u32 dpp_page;
+       bool dpp_enabled;
+};
+
+struct ocrdma_ah {
+       struct ib_ah ibah;
+       struct ocrdma_dev *dev;
+       struct ocrdma_av *av;
+       u16 sgid_index;
+       u32 id;
+};
+
+struct ocrdma_qp_hwq_info {
+       u8 *va;                 /* virtual address */
+       u32 max_sges;
+       u32 head, tail;
+       u32 entry_size;
+       u32 max_cnt;
+       u32 max_wqe_idx;
+       u32 free_delta;
+       u16 dbid;               /* qid, where to ring the doorbell. */
+       u32 len;
+       dma_addr_t pa;
+};
+
+struct ocrdma_srq {
+       struct ib_srq ibsrq;
+       struct ocrdma_dev *dev;
+       u8 __iomem *db;
+       /* provide synchronization to multiple context(s) posting rqe */
+       spinlock_t q_lock ____cacheline_aligned;
+
+       struct ocrdma_qp_hwq_info rq;
+       struct ocrdma_pd *pd;
+       atomic_t use_cnt;
+       u32 id;
+       u64 *rqe_wr_id_tbl;
+       u32 *idx_bit_fields;
+       u32 bit_fields_len;
+};
+
+struct ocrdma_qp {
+       struct ib_qp ibqp;
+       struct ocrdma_dev *dev;
+
+       u8 __iomem *sq_db;
+       /* provide synchronization to multiple context(s) posting wqe, rqe */
+       spinlock_t q_lock ____cacheline_aligned;
+       struct ocrdma_qp_hwq_info sq;
+       struct {
+               uint64_t wrid;
+               uint16_t dpp_wqe_idx;
+               uint16_t dpp_wqe;
+               uint8_t  signaled;
+               uint8_t  rsvd[3];
+       } *wqe_wr_id_tbl;
+       u32 max_inline_data;
+       struct ocrdma_cq *sq_cq;
+       /* list maintained per CQ to flush SQ errors */
+       struct list_head sq_entry;
+
+       u8 __iomem *rq_db;
+       struct ocrdma_qp_hwq_info rq;
+       u64 *rqe_wr_id_tbl;
+       struct ocrdma_cq *rq_cq;
+       struct ocrdma_srq *srq;
+       /* list maintained per CQ to flush RQ errors */
+       struct list_head rq_entry;
+
+       enum ocrdma_qp_state state;     /*  QP state */
+       int cap_flags;
+       u32 max_ord, max_ird;
+
+       u32 id;
+       struct ocrdma_pd *pd;
+
+       enum ib_qp_type qp_type;
+
+       int sgid_idx;
+       u32 qkey;
+       bool dpp_enabled;
+       u8 *ird_q_va;
+};
+
+#define OCRDMA_GET_NUM_POSTED_SHIFT_VAL(qp) \
+       (((qp->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) && \
+               (qp->id < 64)) ? 24 : 16)
+
+struct ocrdma_hw_mr {
+       struct ocrdma_dev *dev;
+       u32 lkey;
+       u8 fr_mr;
+       u8 remote_atomic;
+       u8 remote_rd;
+       u8 remote_wr;
+       u8 local_rd;
+       u8 local_wr;
+       u8 mw_bind;
+       u8 rsvd;
+       u64 len;
+       struct ocrdma_pbl *pbl_table;
+       u32 num_pbls;
+       u32 num_pbes;
+       u32 pbl_size;
+       u32 pbe_size;
+       u64 fbo;
+       u64 va;
+};
+
+struct ocrdma_mr {
+       struct ib_mr ibmr;
+       struct ib_umem *umem;
+       struct ocrdma_hw_mr hwmr;
+       struct ocrdma_pd *pd;
+};
+
+struct ocrdma_ucontext {
+       struct ib_ucontext ibucontext;
+       struct ocrdma_dev *dev;
+
+       struct list_head mm_head;
+       struct mutex mm_list_lock; /* protects list entries of mm type */
+       struct {
+               u32 *va;
+               dma_addr_t pa;
+               u32 len;
+       } ah_tbl;
+};
+
+struct ocrdma_mm {
+       struct {
+               u64 phy_addr;
+               unsigned long len;
+       } key;
+       struct list_head entry;
+};
+
+static inline struct ocrdma_dev *get_ocrdma_dev(struct ib_device *ibdev)
+{
+       return container_of(ibdev, struct ocrdma_dev, ibdev);
+}
+
+static inline struct ocrdma_ucontext *get_ocrdma_ucontext(struct ib_ucontext
+                                                         *ibucontext)
+{
+       return container_of(ibucontext, struct ocrdma_ucontext, ibucontext);
+}
+
+static inline struct ocrdma_pd *get_ocrdma_pd(struct ib_pd *ibpd)
+{
+       return container_of(ibpd, struct ocrdma_pd, ibpd);
+}
+
+static inline struct ocrdma_cq *get_ocrdma_cq(struct ib_cq *ibcq)
+{
+       return container_of(ibcq, struct ocrdma_cq, ibcq);
+}
+
+static inline struct ocrdma_qp *get_ocrdma_qp(struct ib_qp *ibqp)
+{
+       return container_of(ibqp, struct ocrdma_qp, ibqp);
+}
+
+static inline struct ocrdma_mr *get_ocrdma_mr(struct ib_mr *ibmr)
+{
+       return container_of(ibmr, struct ocrdma_mr, ibmr);
+}
+
+static inline struct ocrdma_ah *get_ocrdma_ah(struct ib_ah *ibah)
+{
+       return container_of(ibah, struct ocrdma_ah, ibah);
+}
+
+static inline struct ocrdma_srq *get_ocrdma_srq(struct ib_srq *ibsrq)
+{
+       return container_of(ibsrq, struct ocrdma_srq, ibsrq);
+}
+
+#endif
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_abi.h b/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
new file mode 100644 (file)
index 0000000..a411a4e
--- /dev/null
@@ -0,0 +1,134 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#ifndef __OCRDMA_ABI_H__
+#define __OCRDMA_ABI_H__
+
+struct ocrdma_alloc_ucontext_resp {
+       u32 dev_id;
+       u32 wqe_size;
+       u32 max_inline_data;
+       u32 dpp_wqe_size;
+       u64 ah_tbl_page;
+       u32 ah_tbl_len;
+       u32 rsvd;
+       u8 fw_ver[32];
+       u32 rqe_size;
+       u64 rsvd1;
+} __packed;
+
+/* user kernel communication data structures. */
+struct ocrdma_alloc_pd_ureq {
+       u64 rsvd1;
+} __packed;
+
+struct ocrdma_alloc_pd_uresp {
+       u32 id;
+       u32 dpp_enabled;
+       u32 dpp_page_addr_hi;
+       u32 dpp_page_addr_lo;
+       u64 rsvd1;
+} __packed;
+
+struct ocrdma_create_cq_ureq {
+       u32 dpp_cq;
+       u32 rsvd;
+} __packed;
+
+#define MAX_CQ_PAGES 8
+struct ocrdma_create_cq_uresp {
+       u32 cq_id;
+       u32 page_size;
+       u32 num_pages;
+       u32 max_hw_cqe;
+       u64 page_addr[MAX_CQ_PAGES];
+       u64 db_page_addr;
+       u32 db_page_size;
+       u32 phase_change;
+       u64 rsvd1;
+       u64 rsvd2;
+} __packed;
+
+#define MAX_QP_PAGES 8
+#define MAX_UD_AV_PAGES 8
+
+struct ocrdma_create_qp_ureq {
+       u8 enable_dpp_cq;
+       u8 rsvd;
+       u16 dpp_cq_id;
+       u32 rsvd1;
+};
+
+struct ocrdma_create_qp_uresp {
+       u16 qp_id;
+       u16 sq_dbid;
+       u16 rq_dbid;
+       u16 resv0;
+       u32 sq_page_size;
+       u32 rq_page_size;
+       u32 num_sq_pages;
+       u32 num_rq_pages;
+       u64 sq_page_addr[MAX_QP_PAGES];
+       u64 rq_page_addr[MAX_QP_PAGES];
+       u64 db_page_addr;
+       u32 db_page_size;
+       u32 dpp_credit;
+       u32 dpp_offset;
+       u32 rsvd1;
+       u32 num_wqe_allocated;
+       u32 num_rqe_allocated;
+       u32 free_wqe_delta;
+       u32 free_rqe_delta;
+       u32 db_sq_offset;
+       u32 db_rq_offset;
+       u32 db_shift;
+       u64 rsvd2;
+       u64 rsvd3;
+} __packed;
+
+struct ocrdma_create_srq_uresp {
+       u16 rq_dbid;
+       u16 resv0;
+       u32 resv1;
+
+       u32 rq_page_size;
+       u32 num_rq_pages;
+
+       u64 rq_page_addr[MAX_QP_PAGES];
+       u64 db_page_addr;
+
+       u32 db_page_size;
+       u32 num_rqe_allocated;
+       u32 db_rq_offset;
+       u32 db_shift;
+
+       u32 free_rqe_delta;
+       u32 rsvd2;
+       u64 rsvd3;
+} __packed;
+
+#endif                         /* __OCRDMA_ABI_H__ */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
new file mode 100644 (file)
index 0000000..a877a8e
--- /dev/null
@@ -0,0 +1,172 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#include <net/neighbour.h>
+#include <net/netevent.h>
+
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+
+#include "ocrdma.h"
+#include "ocrdma_verbs.h"
+#include "ocrdma_ah.h"
+#include "ocrdma_hw.h"
+
+static inline int set_av_attr(struct ocrdma_ah *ah,
+                               struct ib_ah_attr *attr, int pdid)
+{
+       int status = 0;
+       u16 vlan_tag; bool vlan_enabled = false;
+       struct ocrdma_dev *dev = ah->dev;
+       struct ocrdma_eth_vlan eth;
+       struct ocrdma_grh grh;
+       int eth_sz;
+
+       memset(&eth, 0, sizeof(eth));
+       memset(&grh, 0, sizeof(grh));
+
+       ah->sgid_index = attr->grh.sgid_index;
+
+       vlan_tag = rdma_get_vlan_id(&attr->grh.dgid);
+       if (vlan_tag && (vlan_tag < 0x1000)) {
+               eth.eth_type = cpu_to_be16(0x8100);
+               eth.roce_eth_type = cpu_to_be16(OCRDMA_ROCE_ETH_TYPE);
+               vlan_tag |= (attr->sl & 7) << 13;
+               eth.vlan_tag = cpu_to_be16(vlan_tag);
+               eth_sz = sizeof(struct ocrdma_eth_vlan);
+               vlan_enabled = true;
+       } else {
+               eth.eth_type = cpu_to_be16(OCRDMA_ROCE_ETH_TYPE);
+               eth_sz = sizeof(struct ocrdma_eth_basic);
+       }
+       memcpy(&eth.smac[0], &dev->nic_info.mac_addr[0], ETH_ALEN);
+       status = ocrdma_resolve_dgid(dev, &attr->grh.dgid, &eth.dmac[0]);
+       if (status)
+               return status;
+       status = ocrdma_query_gid(&dev->ibdev, 1, attr->grh.sgid_index,
+                       (union ib_gid *)&grh.sgid[0]);
+       if (status)
+               return status;
+
+       grh.tclass_flow = cpu_to_be32((6 << 28) |
+                       (attr->grh.traffic_class << 24) |
+                       attr->grh.flow_label);
+       /* 0x1b is next header value in GRH */
+       grh.pdid_hoplimit = cpu_to_be32((pdid << 16) |
+                       (0x1b << 8) | attr->grh.hop_limit);
+
+       memcpy(&grh.dgid[0], attr->grh.dgid.raw, sizeof(attr->grh.dgid.raw));
+       memcpy(&ah->av->eth_hdr, &eth, eth_sz);
+       memcpy((u8 *)ah->av + eth_sz, &grh, sizeof(struct ocrdma_grh));
+       if (vlan_enabled)
+               ah->av->valid |= OCRDMA_AV_VLAN_VALID;
+       return status;
+}
+
+struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
+{
+       u32 *ahid_addr;
+       int status;
+       struct ocrdma_ah *ah;
+       struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
+       struct ocrdma_dev *dev = pd->dev;
+
+       if (!(attr->ah_flags & IB_AH_GRH))
+               return ERR_PTR(-EINVAL);
+
+       ah = kzalloc(sizeof *ah, GFP_ATOMIC);
+       if (!ah)
+               return ERR_PTR(-ENOMEM);
+       ah->dev = pd->dev;
+
+       status = ocrdma_alloc_av(dev, ah);
+       if (status)
+               goto av_err;
+       status = set_av_attr(ah, attr, pd->id);
+       if (status)
+               goto av_conf_err;
+
+       /* if pd is for the user process, pass the ah_id to user space */
+       if ((pd->uctx) && (pd->uctx->ah_tbl.va)) {
+               ahid_addr = pd->uctx->ah_tbl.va + attr->dlid;
+               *ahid_addr = ah->id;
+       }
+       return &ah->ibah;
+
+av_conf_err:
+       ocrdma_free_av(dev, ah);
+av_err:
+       kfree(ah);
+       return ERR_PTR(status);
+}
+
+int ocrdma_destroy_ah(struct ib_ah *ibah)
+{
+       struct ocrdma_ah *ah = get_ocrdma_ah(ibah);
+       ocrdma_free_av(ah->dev, ah);
+       kfree(ah);
+       return 0;
+}
+
+int ocrdma_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
+{
+       struct ocrdma_ah *ah = get_ocrdma_ah(ibah);
+       struct ocrdma_av *av = ah->av;
+       struct ocrdma_grh *grh;
+       attr->ah_flags |= IB_AH_GRH;
+       if (ah->av->valid & Bit(1)) {
+               grh = (struct ocrdma_grh *)((u8 *)ah->av +
+                               sizeof(struct ocrdma_eth_vlan));
+               attr->sl = be16_to_cpu(av->eth_hdr.vlan_tag) >> 13;
+       } else {
+               grh = (struct ocrdma_grh *)((u8 *)ah->av +
+                                       sizeof(struct ocrdma_eth_basic));
+               attr->sl = 0;
+       }
+       memcpy(&attr->grh.dgid.raw[0], &grh->dgid[0], sizeof(grh->dgid));
+       attr->grh.sgid_index = ah->sgid_index;
+       attr->grh.hop_limit = be32_to_cpu(grh->pdid_hoplimit) & 0xff;
+       attr->grh.traffic_class = be32_to_cpu(grh->tclass_flow) >> 24;
+       attr->grh.flow_label = be32_to_cpu(grh->tclass_flow) & 0x00ffffffff;
+       return 0;
+}
+
+int ocrdma_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
+{
+       /* modify_ah is unsupported */
+       return -ENOSYS;
+}
+
+int ocrdma_process_mad(struct ib_device *ibdev,
+                      int process_mad_flags,
+                      u8 port_num,
+                      struct ib_wc *in_wc,
+                      struct ib_grh *in_grh,
+                      struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+       return IB_MAD_RESULT_SUCCESS;
+}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
new file mode 100644 (file)
index 0000000..8ac49e7
--- /dev/null
@@ -0,0 +1,42 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#ifndef __OCRDMA_AH_H__
+#define __OCRDMA_AH_H__
+
+struct ib_ah *ocrdma_create_ah(struct ib_pd *, struct ib_ah_attr *);
+int ocrdma_destroy_ah(struct ib_ah *);
+int ocrdma_query_ah(struct ib_ah *, struct ib_ah_attr *);
+int ocrdma_modify_ah(struct ib_ah *, struct ib_ah_attr *);
+
+int ocrdma_process_mad(struct ib_device *,
+                      int process_mad_flags,
+                      u8 port_num,
+                      struct ib_wc *in_wc,
+                      struct ib_grh *in_grh,
+                      struct ib_mad *in_mad, struct ib_mad *out_mad);
+#endif                         /* __OCRDMA_AH_H__ */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
new file mode 100644 (file)
index 0000000..9b204b1
--- /dev/null
@@ -0,0 +1,2640 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) CNA Adapters.              *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/log2.h>
+#include <linux/dma-mapping.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+
+#include "ocrdma.h"
+#include "ocrdma_hw.h"
+#include "ocrdma_verbs.h"
+#include "ocrdma_ah.h"
+
+enum mbx_status {
+       OCRDMA_MBX_STATUS_FAILED                = 1,
+       OCRDMA_MBX_STATUS_ILLEGAL_FIELD         = 3,
+       OCRDMA_MBX_STATUS_OOR                   = 100,
+       OCRDMA_MBX_STATUS_INVALID_PD            = 101,
+       OCRDMA_MBX_STATUS_PD_INUSE              = 102,
+       OCRDMA_MBX_STATUS_INVALID_CQ            = 103,
+       OCRDMA_MBX_STATUS_INVALID_QP            = 104,
+       OCRDMA_MBX_STATUS_INVALID_LKEY          = 105,
+       OCRDMA_MBX_STATUS_ORD_EXCEEDS           = 106,
+       OCRDMA_MBX_STATUS_IRD_EXCEEDS           = 107,
+       OCRDMA_MBX_STATUS_SENDQ_WQE_EXCEEDS     = 108,
+       OCRDMA_MBX_STATUS_RECVQ_RQE_EXCEEDS     = 109,
+       OCRDMA_MBX_STATUS_SGE_SEND_EXCEEDS      = 110,
+       OCRDMA_MBX_STATUS_SGE_WRITE_EXCEEDS     = 111,
+       OCRDMA_MBX_STATUS_SGE_RECV_EXCEEDS      = 112,
+       OCRDMA_MBX_STATUS_INVALID_STATE_CHANGE  = 113,
+       OCRDMA_MBX_STATUS_MW_BOUND              = 114,
+       OCRDMA_MBX_STATUS_INVALID_VA            = 115,
+       OCRDMA_MBX_STATUS_INVALID_LENGTH        = 116,
+       OCRDMA_MBX_STATUS_INVALID_FBO           = 117,
+       OCRDMA_MBX_STATUS_INVALID_ACC_RIGHTS    = 118,
+       OCRDMA_MBX_STATUS_INVALID_PBE_SIZE      = 119,
+       OCRDMA_MBX_STATUS_INVALID_PBL_ENTRY     = 120,
+       OCRDMA_MBX_STATUS_INVALID_PBL_SHIFT     = 121,
+       OCRDMA_MBX_STATUS_INVALID_SRQ_ID        = 129,
+       OCRDMA_MBX_STATUS_SRQ_ERROR             = 133,
+       OCRDMA_MBX_STATUS_RQE_EXCEEDS           = 134,
+       OCRDMA_MBX_STATUS_MTU_EXCEEDS           = 135,
+       OCRDMA_MBX_STATUS_MAX_QP_EXCEEDS        = 136,
+       OCRDMA_MBX_STATUS_SRQ_LIMIT_EXCEEDS     = 137,
+       OCRDMA_MBX_STATUS_SRQ_SIZE_UNDERUNS     = 138,
+       OCRDMA_MBX_STATUS_QP_BOUND              = 130,
+       OCRDMA_MBX_STATUS_INVALID_CHANGE        = 139,
+       OCRDMA_MBX_STATUS_ATOMIC_OPS_UNSUP      = 140,
+       OCRDMA_MBX_STATUS_INVALID_RNR_NAK_TIMER = 141,
+       OCRDMA_MBX_STATUS_MW_STILL_BOUND        = 142,
+       OCRDMA_MBX_STATUS_PKEY_INDEX_INVALID    = 143,
+       OCRDMA_MBX_STATUS_PKEY_INDEX_EXCEEDS    = 144
+};
+
+enum additional_status {
+       OCRDMA_MBX_ADDI_STATUS_INSUFFICIENT_RESOURCES = 22
+};
+
+enum cqe_status {
+       OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_PRIVILEDGES  = 1,
+       OCRDMA_MBX_CQE_STATUS_INVALID_PARAMETER         = 2,
+       OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_RESOURCES    = 3,
+       OCRDMA_MBX_CQE_STATUS_QUEUE_FLUSHING            = 4,
+       OCRDMA_MBX_CQE_STATUS_DMA_FAILED                = 5
+};
+
+static inline void *ocrdma_get_eqe(struct ocrdma_eq *eq)
+{
+       return (u8 *)eq->q.va + (eq->q.tail * sizeof(struct ocrdma_eqe));
+}
+
+static inline void ocrdma_eq_inc_tail(struct ocrdma_eq *eq)
+{
+       eq->q.tail = (eq->q.tail + 1) & (OCRDMA_EQ_LEN - 1);
+}
+
+static inline void *ocrdma_get_mcqe(struct ocrdma_dev *dev)
+{
+       struct ocrdma_mcqe *cqe = (struct ocrdma_mcqe *)
+           ((u8 *) dev->mq.cq.va +
+            (dev->mq.cq.tail * sizeof(struct ocrdma_mcqe)));
+
+       if (!(le32_to_cpu(cqe->valid_ae_cmpl_cons) & OCRDMA_MCQE_VALID_MASK))
+               return NULL;
+       return cqe;
+}
+
+static inline void ocrdma_mcq_inc_tail(struct ocrdma_dev *dev)
+{
+       dev->mq.cq.tail = (dev->mq.cq.tail + 1) & (OCRDMA_MQ_CQ_LEN - 1);
+}
+
+static inline struct ocrdma_mqe *ocrdma_get_mqe(struct ocrdma_dev *dev)
+{
+       return (struct ocrdma_mqe *)((u8 *) dev->mq.sq.va +
+                                    (dev->mq.sq.head *
+                                     sizeof(struct ocrdma_mqe)));
+}
+
+static inline void ocrdma_mq_inc_head(struct ocrdma_dev *dev)
+{
+       dev->mq.sq.head = (dev->mq.sq.head + 1) & (OCRDMA_MQ_LEN - 1);
+       atomic_inc(&dev->mq.sq.used);
+}
+
+static inline void *ocrdma_get_mqe_rsp(struct ocrdma_dev *dev)
+{
+       return (void *)((u8 *) dev->mq.sq.va +
+                       (dev->mqe_ctx.tag * sizeof(struct ocrdma_mqe)));
+}
+
+enum ib_qp_state get_ibqp_state(enum ocrdma_qp_state qps)
+{
+       switch (qps) {
+       case OCRDMA_QPS_RST:
+               return IB_QPS_RESET;
+       case OCRDMA_QPS_INIT:
+               return IB_QPS_INIT;
+       case OCRDMA_QPS_RTR:
+               return IB_QPS_RTR;
+       case OCRDMA_QPS_RTS:
+               return IB_QPS_RTS;
+       case OCRDMA_QPS_SQD:
+       case OCRDMA_QPS_SQ_DRAINING:
+               return IB_QPS_SQD;
+       case OCRDMA_QPS_SQE:
+               return IB_QPS_SQE;
+       case OCRDMA_QPS_ERR:
+               return IB_QPS_ERR;
+       };
+       return IB_QPS_ERR;
+}
+
+static enum ocrdma_qp_state get_ocrdma_qp_state(enum ib_qp_state qps)
+{
+       switch (qps) {
+       case IB_QPS_RESET:
+               return OCRDMA_QPS_RST;
+       case IB_QPS_INIT:
+               return OCRDMA_QPS_INIT;
+       case IB_QPS_RTR:
+               return OCRDMA_QPS_RTR;
+       case IB_QPS_RTS:
+               return OCRDMA_QPS_RTS;
+       case IB_QPS_SQD:
+               return OCRDMA_QPS_SQD;
+       case IB_QPS_SQE:
+               return OCRDMA_QPS_SQE;
+       case IB_QPS_ERR:
+               return OCRDMA_QPS_ERR;
+       };
+       return OCRDMA_QPS_ERR;
+}
+
+static int ocrdma_get_mbx_errno(u32 status)
+{
+       int err_num = -EFAULT;
+       u8 mbox_status = (status & OCRDMA_MBX_RSP_STATUS_MASK) >>
+                                       OCRDMA_MBX_RSP_STATUS_SHIFT;
+       u8 add_status = (status & OCRDMA_MBX_RSP_ASTATUS_MASK) >>
+                                       OCRDMA_MBX_RSP_ASTATUS_SHIFT;
+
+       switch (mbox_status) {
+       case OCRDMA_MBX_STATUS_OOR:
+       case OCRDMA_MBX_STATUS_MAX_QP_EXCEEDS:
+               err_num = -EAGAIN;
+               break;
+
+       case OCRDMA_MBX_STATUS_INVALID_PD:
+       case OCRDMA_MBX_STATUS_INVALID_CQ:
+       case OCRDMA_MBX_STATUS_INVALID_SRQ_ID:
+       case OCRDMA_MBX_STATUS_INVALID_QP:
+       case OCRDMA_MBX_STATUS_INVALID_CHANGE:
+       case OCRDMA_MBX_STATUS_MTU_EXCEEDS:
+       case OCRDMA_MBX_STATUS_INVALID_RNR_NAK_TIMER:
+       case OCRDMA_MBX_STATUS_PKEY_INDEX_INVALID:
+       case OCRDMA_MBX_STATUS_PKEY_INDEX_EXCEEDS:
+       case OCRDMA_MBX_STATUS_ILLEGAL_FIELD:
+       case OCRDMA_MBX_STATUS_INVALID_PBL_ENTRY:
+       case OCRDMA_MBX_STATUS_INVALID_LKEY:
+       case OCRDMA_MBX_STATUS_INVALID_VA:
+       case OCRDMA_MBX_STATUS_INVALID_LENGTH:
+       case OCRDMA_MBX_STATUS_INVALID_FBO:
+       case OCRDMA_MBX_STATUS_INVALID_ACC_RIGHTS:
+       case OCRDMA_MBX_STATUS_INVALID_PBE_SIZE:
+       case OCRDMA_MBX_STATUS_ATOMIC_OPS_UNSUP:
+       case OCRDMA_MBX_STATUS_SRQ_ERROR:
+       case OCRDMA_MBX_STATUS_SRQ_SIZE_UNDERUNS:
+               err_num = -EINVAL;
+               break;
+
+       case OCRDMA_MBX_STATUS_PD_INUSE:
+       case OCRDMA_MBX_STATUS_QP_BOUND:
+       case OCRDMA_MBX_STATUS_MW_STILL_BOUND:
+       case OCRDMA_MBX_STATUS_MW_BOUND:
+               err_num = -EBUSY;
+               break;
+
+       case OCRDMA_MBX_STATUS_RECVQ_RQE_EXCEEDS:
+       case OCRDMA_MBX_STATUS_SGE_RECV_EXCEEDS:
+       case OCRDMA_MBX_STATUS_RQE_EXCEEDS:
+       case OCRDMA_MBX_STATUS_SRQ_LIMIT_EXCEEDS:
+       case OCRDMA_MBX_STATUS_ORD_EXCEEDS:
+       case OCRDMA_MBX_STATUS_IRD_EXCEEDS:
+       case OCRDMA_MBX_STATUS_SENDQ_WQE_EXCEEDS:
+       case OCRDMA_MBX_STATUS_SGE_SEND_EXCEEDS:
+       case OCRDMA_MBX_STATUS_SGE_WRITE_EXCEEDS:
+               err_num = -ENOBUFS;
+               break;
+
+       case OCRDMA_MBX_STATUS_FAILED:
+               switch (add_status) {
+               case OCRDMA_MBX_ADDI_STATUS_INSUFFICIENT_RESOURCES:
+                       err_num = -EAGAIN;
+                       break;
+               }
+       default:
+               err_num = -EFAULT;
+       }
+       return err_num;
+}
+
+static int ocrdma_get_mbx_cqe_errno(u16 cqe_status)
+{
+       int err_num = -EINVAL;
+
+       switch (cqe_status) {
+       case OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_PRIVILEDGES:
+               err_num = -EPERM;
+               break;
+       case OCRDMA_MBX_CQE_STATUS_INVALID_PARAMETER:
+               err_num = -EINVAL;
+               break;
+       case OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_RESOURCES:
+       case OCRDMA_MBX_CQE_STATUS_QUEUE_FLUSHING:
+               err_num = -EAGAIN;
+               break;
+       case OCRDMA_MBX_CQE_STATUS_DMA_FAILED:
+               err_num = -EIO;
+               break;
+       }
+       return err_num;
+}
+
+void ocrdma_ring_cq_db(struct ocrdma_dev *dev, u16 cq_id, bool armed,
+                      bool solicited, u16 cqe_popped)
+{
+       u32 val = cq_id & OCRDMA_DB_CQ_RING_ID_MASK;
+
+       val |= ((cq_id & OCRDMA_DB_CQ_RING_ID_EXT_MASK) <<
+            OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT);
+
+       if (armed)
+               val |= (1 << OCRDMA_DB_CQ_REARM_SHIFT);
+       if (solicited)
+               val |= (1 << OCRDMA_DB_CQ_SOLICIT_SHIFT);
+       val |= (cqe_popped << OCRDMA_DB_CQ_NUM_POPPED_SHIFT);
+       iowrite32(val, dev->nic_info.db + OCRDMA_DB_CQ_OFFSET);
+}
+
+static void ocrdma_ring_mq_db(struct ocrdma_dev *dev)
+{
+       u32 val = 0;
+
+       val |= dev->mq.sq.id & OCRDMA_MQ_ID_MASK;
+       val |= 1 << OCRDMA_MQ_NUM_MQE_SHIFT;
+       iowrite32(val, dev->nic_info.db + OCRDMA_DB_MQ_OFFSET);
+}
+
+static void ocrdma_ring_eq_db(struct ocrdma_dev *dev, u16 eq_id,
+                             bool arm, bool clear_int, u16 num_eqe)
+{
+       u32 val = 0;
+
+       val |= eq_id & OCRDMA_EQ_ID_MASK;
+       val |= ((eq_id & OCRDMA_EQ_ID_EXT_MASK) << OCRDMA_EQ_ID_EXT_MASK_SHIFT);
+       if (arm)
+               val |= (1 << OCRDMA_REARM_SHIFT);
+       if (clear_int)
+               val |= (1 << OCRDMA_EQ_CLR_SHIFT);
+       val |= (1 << OCRDMA_EQ_TYPE_SHIFT);
+       val |= (num_eqe << OCRDMA_NUM_EQE_SHIFT);
+       iowrite32(val, dev->nic_info.db + OCRDMA_DB_EQ_OFFSET);
+}
+
+static void ocrdma_init_mch(struct ocrdma_mbx_hdr *cmd_hdr,
+                           u8 opcode, u8 subsys, u32 cmd_len)
+{
+       cmd_hdr->subsys_op = (opcode | (subsys << OCRDMA_MCH_SUBSYS_SHIFT));
+       cmd_hdr->timeout = 20; /* seconds */
+       cmd_hdr->cmd_len = cmd_len - sizeof(struct ocrdma_mbx_hdr);
+}
+
+static void *ocrdma_init_emb_mqe(u8 opcode, u32 cmd_len)
+{
+       struct ocrdma_mqe *mqe;
+
+       mqe = kzalloc(sizeof(struct ocrdma_mqe), GFP_KERNEL);
+       if (!mqe)
+               return NULL;
+       mqe->hdr.spcl_sge_cnt_emb |=
+               (OCRDMA_MQE_EMBEDDED << OCRDMA_MQE_HDR_EMB_SHIFT) &
+                                       OCRDMA_MQE_HDR_EMB_MASK;
+       mqe->hdr.pyld_len = cmd_len - sizeof(struct ocrdma_mqe_hdr);
+
+       ocrdma_init_mch(&mqe->u.emb_req.mch, opcode, OCRDMA_SUBSYS_ROCE,
+                       mqe->hdr.pyld_len);
+       return mqe;
+}
+
+static void ocrdma_free_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q)
+{
+       dma_free_coherent(&dev->nic_info.pdev->dev, q->size, q->va, q->dma);
+}
+
+static int ocrdma_alloc_q(struct ocrdma_dev *dev,
+                         struct ocrdma_queue_info *q, u16 len, u16 entry_size)
+{
+       memset(q, 0, sizeof(*q));
+       q->len = len;
+       q->entry_size = entry_size;
+       q->size = len * entry_size;
+       q->va = dma_alloc_coherent(&dev->nic_info.pdev->dev, q->size,
+                                  &q->dma, GFP_KERNEL);
+       if (!q->va)
+               return -ENOMEM;
+       memset(q->va, 0, q->size);
+       return 0;
+}
+
+static void ocrdma_build_q_pages(struct ocrdma_pa *q_pa, int cnt,
+                                       dma_addr_t host_pa, int hw_page_size)
+{
+       int i;
+
+       for (i = 0; i < cnt; i++) {
+               q_pa[i].lo = (u32) (host_pa & 0xffffffff);
+               q_pa[i].hi = (u32) upper_32_bits(host_pa);
+               host_pa += hw_page_size;
+       }
+}
+
+static void ocrdma_assign_eq_vect_gen2(struct ocrdma_dev *dev,
+                                      struct ocrdma_eq *eq)
+{
+       /* assign vector and update vector id for next EQ */
+       eq->vector = dev->nic_info.msix.start_vector;
+       dev->nic_info.msix.start_vector += 1;
+}
+
+static void ocrdma_free_eq_vect_gen2(struct ocrdma_dev *dev)
+{
+       /* this assumes that EQs are freed in exactly reverse order
+        * as its allocation.
+        */
+       dev->nic_info.msix.start_vector -= 1;
+}
+
+static int ocrdma_mbx_delete_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q,
+                              int queue_type)
+{
+       u8 opcode = 0;
+       int status;
+       struct ocrdma_delete_q_req *cmd = dev->mbx_cmd;
+
+       switch (queue_type) {
+       case QTYPE_MCCQ:
+               opcode = OCRDMA_CMD_DELETE_MQ;
+               break;
+       case QTYPE_CQ:
+               opcode = OCRDMA_CMD_DELETE_CQ;
+               break;
+       case QTYPE_EQ:
+               opcode = OCRDMA_CMD_DELETE_EQ;
+               break;
+       default:
+               BUG();
+       }
+       memset(cmd, 0, sizeof(*cmd));
+       ocrdma_init_mch(&cmd->req, opcode, OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+       cmd->id = q->id;
+
+       status = be_roce_mcc_cmd(dev->nic_info.netdev,
+                                cmd, sizeof(*cmd), NULL, NULL);
+       if (!status)
+               q->created = false;
+       return status;
+}
+
+static int ocrdma_mbx_create_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
+{
+       int status;
+       struct ocrdma_create_eq_req *cmd = dev->mbx_cmd;
+       struct ocrdma_create_eq_rsp *rsp = dev->mbx_cmd;
+
+       memset(cmd, 0, sizeof(*cmd));
+       ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_EQ, OCRDMA_SUBSYS_COMMON,
+                       sizeof(*cmd));
+       if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY)
+               cmd->req.rsvd_version = 0;
+       else
+               cmd->req.rsvd_version = 2;
+
+       cmd->num_pages = 4;
+       cmd->valid = OCRDMA_CREATE_EQ_VALID;
+       cmd->cnt = 4 << OCRDMA_CREATE_EQ_CNT_SHIFT;
+
+       ocrdma_build_q_pages(&cmd->pa[0], cmd->num_pages, eq->q.dma,
+                            PAGE_SIZE_4K);
+       status = be_roce_mcc_cmd(dev->nic_info.netdev, cmd, sizeof(*cmd), NULL,
+                                NULL);
+       if (!status) {
+               eq->q.id = rsp->vector_eqid & 0xffff;
+               if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY)
+                       ocrdma_assign_eq_vect_gen2(dev, eq);
+               else {
+                       eq->vector = (rsp->vector_eqid >> 16) & 0xffff;
+                       dev->nic_info.msix.start_vector += 1;
+               }
+               eq->q.created = true;
+       }
+       return status;
+}
+
+static int ocrdma_create_eq(struct ocrdma_dev *dev,
+                           struct ocrdma_eq *eq, u16 q_len)
+{
+       int status;
+
+       status = ocrdma_alloc_q(dev, &eq->q, OCRDMA_EQ_LEN,
+                               sizeof(struct ocrdma_eqe));
+       if (status)
+               return status;
+
+       status = ocrdma_mbx_create_eq(dev, eq);
+       if (status)
+               goto mbx_err;
+       eq->dev = dev;
+       ocrdma_ring_eq_db(dev, eq->q.id, true, true, 0);
+
+       return 0;
+mbx_err:
+       ocrdma_free_q(dev, &eq->q);
+       return status;
+}
+
+static int ocrdma_get_irq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
+{
+       int irq;
+
+       if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX)
+               irq = dev->nic_info.pdev->irq;
+       else
+               irq = dev->nic_info.msix.vector_list[eq->vector];
+       return irq;
+}
+
+static void _ocrdma_destroy_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
+{
+       if (eq->q.created) {
+               ocrdma_mbx_delete_q(dev, &eq->q, QTYPE_EQ);
+               if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY)
+                       ocrdma_free_eq_vect_gen2(dev);
+               ocrdma_free_q(dev, &eq->q);
+       }
+}
+
+static void ocrdma_destroy_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
+{
+       int irq;
+
+       /* disarm EQ so that interrupts are not generated
+        * during freeing and EQ delete is in progress.
+        */
+       ocrdma_ring_eq_db(dev, eq->q.id, false, false, 0);
+
+       irq = ocrdma_get_irq(dev, eq);
+       free_irq(irq, eq);
+       _ocrdma_destroy_eq(dev, eq);
+}
+
+static void ocrdma_destroy_qp_eqs(struct ocrdma_dev *dev)
+{
+       int i;
+
+       /* deallocate the data path eqs */
+       for (i = 0; i < dev->eq_cnt; i++)
+               ocrdma_destroy_eq(dev, &dev->qp_eq_tbl[i]);
+}
+
+static int ocrdma_mbx_mq_cq_create(struct ocrdma_dev *dev,
+                                  struct ocrdma_queue_info *cq,
+                                  struct ocrdma_queue_info *eq)
+{
+       struct ocrdma_create_cq_cmd *cmd = dev->mbx_cmd;
+       struct ocrdma_create_cq_cmd_rsp *rsp = dev->mbx_cmd;
+       int status;
+
+       memset(cmd, 0, sizeof(*cmd));
+       ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_CQ,
+                       OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+
+       cmd->pgsz_pgcnt = PAGES_4K_SPANNED(cq->va, cq->size);
+       cmd->ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS;
+       cmd->eqn = (eq->id << OCRDMA_CREATE_CQ_EQID_SHIFT);
+
+       ocrdma_build_q_pages(&cmd->pa[0], cmd->pgsz_pgcnt,
+                            cq->dma, PAGE_SIZE_4K);
+       status = be_roce_mcc_cmd(dev->nic_info.netdev,
+                                cmd, sizeof(*cmd), NULL, NULL);
+       if (!status) {
+               cq->id = (rsp->cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK);
+               cq->created = true;
+       }
+       return status;
+}
+
+static u32 ocrdma_encoded_q_len(int q_len)
+{
+       u32 len_encoded = fls(q_len);   /* log2(len) + 1 */
+
+       if (len_encoded == 16)
+               len_encoded = 0;
+       return len_encoded;
+}
+
+static int ocrdma_mbx_create_mq(struct ocrdma_dev *dev,
+                               struct ocrdma_queue_info *mq,
+                               struct ocrdma_queue_info *cq)
+{
+       int num_pages, status;
+       struct ocrdma_create_mq_req *cmd = dev->mbx_cmd;
+       struct ocrdma_create_mq_rsp *rsp = dev->mbx_cmd;
+       struct ocrdma_pa *pa;
+
+       memset(cmd, 0, sizeof(*cmd));
+       num_pages = PAGES_4K_SPANNED(mq->va, mq->size);
+
+       if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
+               ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_MQ,
+                               OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+               cmd->v0.pages = num_pages;
+               cmd->v0.async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID;
+               cmd->v0.async_cqid_valid = (cq->id << 1);
+               cmd->v0.cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) <<
+                                            OCRDMA_CREATE_MQ_RING_SIZE_SHIFT);
+               cmd->v0.cqid_ringsize |=
+                       (cq->id << OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT);
+               cmd->v0.valid = OCRDMA_CREATE_MQ_VALID;
+               pa = &cmd->v0.pa[0];
+       } else {
+               ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_MQ_EXT,
+                               OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+               cmd->req.rsvd_version = 1;
+               cmd->v1.cqid_pages = num_pages;
+               cmd->v1.cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT);
+               cmd->v1.async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID;
+               cmd->v1.async_event_bitmap = Bit(20);
+               cmd->v1.async_cqid_ringsize = cq->id;
+               cmd->v1.async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) <<
+                                            OCRDMA_CREATE_MQ_RING_SIZE_SHIFT);
+               cmd->v1.valid = OCRDMA_CREATE_MQ_VALID;
+               pa = &cmd->v1.pa[0];
+       }
+       ocrdma_build_q_pages(pa, num_pages, mq->dma, PAGE_SIZE_4K);
+       status = be_roce_mcc_cmd(dev->nic_info.netdev,
+                                cmd, sizeof(*cmd), NULL, NULL);
+       if (!status) {
+               mq->id = rsp->id;
+               mq->created = true;
+       }
+       return status;
+}
+
+static int ocrdma_create_mq(struct ocrdma_dev *dev)
+{
+       int status;
+
+       /* Alloc completion queue for Mailbox queue */
+       status = ocrdma_alloc_q(dev, &dev->mq.cq, OCRDMA_MQ_CQ_LEN,
+                               sizeof(struct ocrdma_mcqe));
+       if (status)
+               goto alloc_err;
+
+       status = ocrdma_mbx_mq_cq_create(dev, &dev->mq.cq, &dev->meq.q);
+       if (status)
+               goto mbx_cq_free;
+
+       memset(&dev->mqe_ctx, 0, sizeof(dev->mqe_ctx));
+       init_waitqueue_head(&dev->mqe_ctx.cmd_wait);
+       mutex_init(&dev->mqe_ctx.lock);
+
+       /* Alloc Mailbox queue */
+       status = ocrdma_alloc_q(dev, &dev->mq.sq, OCRDMA_MQ_LEN,
+                               sizeof(struct ocrdma_mqe));
+       if (status)
+               goto mbx_cq_destroy;
+       status = ocrdma_mbx_create_mq(dev, &dev->mq.sq, &dev->mq.cq);
+       if (status)
+               goto mbx_q_free;
+       ocrdma_ring_cq_db(dev, dev->mq.cq.id, true, false, 0);
+       return 0;
+
+mbx_q_free:
+       ocrdma_free_q(dev, &dev->mq.sq);
+mbx_cq_destroy:
+       ocrdma_mbx_delete_q(dev, &dev->mq.cq, QTYPE_CQ);
+mbx_cq_free:
+       ocrdma_free_q(dev, &dev->mq.cq);
+alloc_err:
+       return status;
+}
+
+static void ocrdma_destroy_mq(struct ocrdma_dev *dev)
+{
+       struct ocrdma_queue_info *mbxq, *cq;
+
+       /* mqe_ctx lock synchronizes with any other pending cmds. */
+       mutex_lock(&dev->mqe_ctx.lock);
+       mbxq = &dev->mq.sq;
+       if (mbxq->created) {
+               ocrdma_mbx_delete_q(dev, mbxq, QTYPE_MCCQ);
+               ocrdma_free_q(dev, mbxq);
+       }
+       mutex_unlock(&dev->mqe_ctx.lock);
+
+       cq = &dev->mq.cq;
+       if (cq->created) {
+               ocrdma_mbx_delete_q(dev, cq, QTYPE_CQ);
+               ocrdma_free_q(dev, cq);
+       }
+}
+
+static void ocrdma_process_qpcat_error(struct ocrdma_dev *dev,
+                                      struct ocrdma_qp *qp)
+{
+       enum ib_qp_state new_ib_qps = IB_QPS_ERR;
+       enum ib_qp_state old_ib_qps;
+
+       if (qp == NULL)
+               BUG();
+       ocrdma_qp_state_machine(qp, new_ib_qps, &old_ib_qps);
+}
+
+static void ocrdma_dispatch_ibevent(struct ocrdma_dev *dev,
+                                   struct ocrdma_ae_mcqe *cqe)
+{
+       struct ocrdma_qp *qp = NULL;
+       struct ocrdma_cq *cq = NULL;
+       struct ib_event ib_evt;
+       int cq_event = 0;
+       int qp_event = 1;
+       int srq_event = 0;
+       int dev_event = 0;
+       int type = (cqe->valid_ae_event & OCRDMA_AE_MCQE_EVENT_TYPE_MASK) >>
+           OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT;
+
+       if (cqe->qpvalid_qpid & OCRDMA_AE_MCQE_QPVALID)
+               qp = dev->qp_tbl[cqe->qpvalid_qpid & OCRDMA_AE_MCQE_QPID_MASK];
+       if (cqe->cqvalid_cqid & OCRDMA_AE_MCQE_CQVALID)
+               cq = dev->cq_tbl[cqe->cqvalid_cqid & OCRDMA_AE_MCQE_CQID_MASK];
+
+       ib_evt.device = &dev->ibdev;
+
+       switch (type) {
+       case OCRDMA_CQ_ERROR:
+               ib_evt.element.cq = &cq->ibcq;
+               ib_evt.event = IB_EVENT_CQ_ERR;
+               cq_event = 1;
+               qp_event = 0;
+               break;
+       case OCRDMA_CQ_OVERRUN_ERROR:
+               ib_evt.element.cq = &cq->ibcq;
+               ib_evt.event = IB_EVENT_CQ_ERR;
+               break;
+       case OCRDMA_CQ_QPCAT_ERROR:
+               ib_evt.element.qp = &qp->ibqp;
+               ib_evt.event = IB_EVENT_QP_FATAL;
+               ocrdma_process_qpcat_error(dev, qp);
+               break;
+       case OCRDMA_QP_ACCESS_ERROR:
+               ib_evt.element.qp = &qp->ibqp;
+               ib_evt.event = IB_EVENT_QP_ACCESS_ERR;
+               break;
+       case OCRDMA_QP_COMM_EST_EVENT:
+               ib_evt.element.qp = &qp->ibqp;
+               ib_evt.event = IB_EVENT_COMM_EST;
+               break;
+       case OCRDMA_SQ_DRAINED_EVENT:
+               ib_evt.element.qp = &qp->ibqp;
+               ib_evt.event = IB_EVENT_SQ_DRAINED;
+               break;
+       case OCRDMA_DEVICE_FATAL_EVENT:
+               ib_evt.element.port_num = 1;
+               ib_evt.event = IB_EVENT_DEVICE_FATAL;
+               qp_event = 0;
+               dev_event = 1;
+               break;
+       case OCRDMA_SRQCAT_ERROR:
+               ib_evt.element.srq = &qp->srq->ibsrq;
+               ib_evt.event = IB_EVENT_SRQ_ERR;
+               srq_event = 1;
+               qp_event = 0;
+               break;
+       case OCRDMA_SRQ_LIMIT_EVENT:
+               ib_evt.element.srq = &qp->srq->ibsrq;
+               ib_evt.event = IB_EVENT_QP_LAST_WQE_REACHED;
+               srq_event = 1;
+               qp_event = 0;
+               break;
+       case OCRDMA_QP_LAST_WQE_EVENT:
+               ib_evt.element.qp = &qp->ibqp;
+               ib_evt.event = IB_EVENT_QP_LAST_WQE_REACHED;
+               break;
+       default:
+               cq_event = 0;
+               qp_event = 0;
+               srq_event = 0;
+               dev_event = 0;
+               ocrdma_err("%s() unknown type=0x%x\n", __func__, type);
+               break;
+       }
+
+       if (qp_event) {
+               if (qp->ibqp.event_handler)
+                       qp->ibqp.event_handler(&ib_evt, qp->ibqp.qp_context);
+       } else if (cq_event) {
+               if (cq->ibcq.event_handler)
+                       cq->ibcq.event_handler(&ib_evt, cq->ibcq.cq_context);
+       } else if (srq_event) {
+               if (qp->srq->ibsrq.event_handler)
+                       qp->srq->ibsrq.event_handler(&ib_evt,
+                                                    qp->srq->ibsrq.
+                                                    srq_context);
+       } else if (dev_event)
+               ib_dispatch_event(&ib_evt);
+
+}
+
+static void ocrdma_process_acqe(struct ocrdma_dev *dev, void *ae_cqe)
+{
+       /* async CQE processing */
+       struct ocrdma_ae_mcqe *cqe = ae_cqe;
+       u32 evt_code = (cqe->valid_ae_event & OCRDMA_AE_MCQE_EVENT_CODE_MASK) >>
+                       OCRDMA_AE_MCQE_EVENT_CODE_SHIFT;
+
+       if (evt_code == OCRDMA_ASYNC_EVE_CODE)
+               ocrdma_dispatch_ibevent(dev, cqe);
+       else
+               ocrdma_err("%s(%d) invalid evt code=0x%x\n",
+                          __func__, dev->id, evt_code);
+}
+
+static void ocrdma_process_mcqe(struct ocrdma_dev *dev, struct ocrdma_mcqe *cqe)
+{
+       if (dev->mqe_ctx.tag == cqe->tag_lo && dev->mqe_ctx.cmd_done == false) {
+               dev->mqe_ctx.cqe_status = (cqe->status &
+                    OCRDMA_MCQE_STATUS_MASK) >> OCRDMA_MCQE_STATUS_SHIFT;
+               dev->mqe_ctx.ext_status =
+                   (cqe->status & OCRDMA_MCQE_ESTATUS_MASK)
+                   >> OCRDMA_MCQE_ESTATUS_SHIFT;
+               dev->mqe_ctx.cmd_done = true;
+               wake_up(&dev->mqe_ctx.cmd_wait);
+       } else
+               ocrdma_err("%s() cqe for invalid tag0x%x.expected=0x%x\n",
+                          __func__, cqe->tag_lo, dev->mqe_ctx.tag);
+}
+
+static int ocrdma_mq_cq_handler(struct ocrdma_dev *dev, u16 cq_id)
+{
+       u16 cqe_popped = 0;
+       struct ocrdma_mcqe *cqe;
+
+       while (1) {
+               cqe = ocrdma_get_mcqe(dev);
+               if (cqe == NULL)
+                       break;
+               ocrdma_le32_to_cpu(cqe, sizeof(*cqe));
+               cqe_popped += 1;
+               if (cqe->valid_ae_cmpl_cons & OCRDMA_MCQE_AE_MASK)
+                       ocrdma_process_acqe(dev, cqe);
+               else if (cqe->valid_ae_cmpl_cons & OCRDMA_MCQE_CMPL_MASK)
+                       ocrdma_process_mcqe(dev, cqe);
+               else
+                       ocrdma_err("%s() cqe->compl is not set.\n", __func__);
+               memset(cqe, 0, sizeof(struct ocrdma_mcqe));
+               ocrdma_mcq_inc_tail(dev);
+       }
+       ocrdma_ring_cq_db(dev, dev->mq.cq.id, true, false, cqe_popped);
+       return 0;
+}
+
+static void ocrdma_qp_buddy_cq_handler(struct ocrdma_dev *dev,
+                                      struct ocrdma_cq *cq)
+{
+       unsigned long flags;
+       struct ocrdma_qp *qp;
+       bool buddy_cq_found = false;
+       /* Go through list of QPs in error state which are using this CQ
+        * and invoke its callback handler to trigger CQE processing for
+        * error/flushed CQE. It is rare to find more than few entries in
+        * this list as most consumers stops after getting error CQE.
+        * List is traversed only once when a matching buddy cq found for a QP.
+        */
+       spin_lock_irqsave(&dev->flush_q_lock, flags);
+       list_for_each_entry(qp, &cq->sq_head, sq_entry) {
+               if (qp->srq)
+                       continue;
+               /* if wq and rq share the same cq, than comp_handler
+                * is already invoked.
+                */
+               if (qp->sq_cq == qp->rq_cq)
+                       continue;
+               /* if completion came on sq, rq's cq is buddy cq.
+                * if completion came on rq, sq's cq is buddy cq.
+                */
+               if (qp->sq_cq == cq)
+                       cq = qp->rq_cq;
+               else
+                       cq = qp->sq_cq;
+               buddy_cq_found = true;
+               break;
+       }
+       spin_unlock_irqrestore(&dev->flush_q_lock, flags);
+       if (buddy_cq_found == false)
+               return;
+       if (cq->ibcq.comp_handler) {
+               spin_lock_irqsave(&cq->comp_handler_lock, flags);
+               (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+               spin_unlock_irqrestore(&cq->comp_handler_lock, flags);
+       }
+}
+
+static void ocrdma_qp_cq_handler(struct ocrdma_dev *dev, u16 cq_idx)
+{
+       unsigned long flags;
+       struct ocrdma_cq *cq;
+
+       if (cq_idx >= OCRDMA_MAX_CQ)
+               BUG();
+
+       cq = dev->cq_tbl[cq_idx];
+       if (cq == NULL) {
+               ocrdma_err("%s%d invalid id=0x%x\n", __func__, dev->id, cq_idx);
+               return;
+       }
+       spin_lock_irqsave(&cq->cq_lock, flags);
+       cq->armed = false;
+       cq->solicited = false;
+       spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+       ocrdma_ring_cq_db(dev, cq->id, false, false, 0);
+
+       if (cq->ibcq.comp_handler) {
+               spin_lock_irqsave(&cq->comp_handler_lock, flags);
+               (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+               spin_unlock_irqrestore(&cq->comp_handler_lock, flags);
+       }
+       ocrdma_qp_buddy_cq_handler(dev, cq);
+}
+
+static void ocrdma_cq_handler(struct ocrdma_dev *dev, u16 cq_id)
+{
+       /* process the MQ-CQE. */
+       if (cq_id == dev->mq.cq.id)
+               ocrdma_mq_cq_handler(dev, cq_id);
+       else
+               ocrdma_qp_cq_handler(dev, cq_id);
+}
+
+static irqreturn_t ocrdma_irq_handler(int irq, void *handle)
+{
+       struct ocrdma_eq *eq = handle;
+       struct ocrdma_dev *dev = eq->dev;
+       struct ocrdma_eqe eqe;
+       struct ocrdma_eqe *ptr;
+       u16 eqe_popped = 0;
+       u16 cq_id;
+       while (1) {
+               ptr = ocrdma_get_eqe(eq);
+               eqe = *ptr;
+               ocrdma_le32_to_cpu(&eqe, sizeof(eqe));
+               if ((eqe.id_valid & OCRDMA_EQE_VALID_MASK) == 0)
+                       break;
+               eqe_popped += 1;
+               ptr->id_valid = 0;
+               /* check whether its CQE or not. */
+               if ((eqe.id_valid & OCRDMA_EQE_FOR_CQE_MASK) == 0) {
+                       cq_id = eqe.id_valid >> OCRDMA_EQE_RESOURCE_ID_SHIFT;
+                       ocrdma_cq_handler(dev, cq_id);
+               }
+               ocrdma_eq_inc_tail(eq);
+       }
+       ocrdma_ring_eq_db(dev, eq->q.id, true, true, eqe_popped);
+       /* Ring EQ doorbell with num_popped to 0 to enable interrupts again. */
+       if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX)
+               ocrdma_ring_eq_db(dev, eq->q.id, true, true, 0);
+       return IRQ_HANDLED;
+}
+
+static void ocrdma_post_mqe(struct ocrdma_dev *dev, struct ocrdma_mqe *cmd)
+{
+       struct ocrdma_mqe *mqe;
+
+       dev->mqe_ctx.tag = dev->mq.sq.head;
+       dev->mqe_ctx.cmd_done = false;
+       mqe = ocrdma_get_mqe(dev);
+       cmd->hdr.tag_lo = dev->mq.sq.head;
+       ocrdma_copy_cpu_to_le32(mqe, cmd, sizeof(*mqe));
+       /* make sure descriptor is written before ringing doorbell */
+       wmb();
+       ocrdma_mq_inc_head(dev);
+       ocrdma_ring_mq_db(dev);
+}
+
+static int ocrdma_wait_mqe_cmpl(struct ocrdma_dev *dev)
+{
+       long status;
+       /* 30 sec timeout */
+       status = wait_event_timeout(dev->mqe_ctx.cmd_wait,
+                                   (dev->mqe_ctx.cmd_done != false),
+                                   msecs_to_jiffies(30000));
+       if (status)
+               return 0;
+       else
+               return -1;
+}
+
+/* issue a mailbox command on the MQ */
+static int ocrdma_mbx_cmd(struct ocrdma_dev *dev, struct ocrdma_mqe *mqe)
+{
+       int status = 0;
+       u16 cqe_status, ext_status;
+       struct ocrdma_mqe *rsp;
+
+       mutex_lock(&dev->mqe_ctx.lock);
+       ocrdma_post_mqe(dev, mqe);
+       status = ocrdma_wait_mqe_cmpl(dev);
+       if (status)
+               goto mbx_err;
+       cqe_status = dev->mqe_ctx.cqe_status;
+       ext_status = dev->mqe_ctx.ext_status;
+       rsp = ocrdma_get_mqe_rsp(dev);
+       ocrdma_copy_le32_to_cpu(mqe, rsp, (sizeof(*mqe)));
+       if (cqe_status || ext_status) {
+               ocrdma_err
+                   ("%s() opcode=0x%x, cqe_status=0x%x, ext_status=0x%x\n",
+                    __func__,
+                    (rsp->u.rsp.subsys_op & OCRDMA_MBX_RSP_OPCODE_MASK) >>
+                    OCRDMA_MBX_RSP_OPCODE_SHIFT, cqe_status, ext_status);
+               status = ocrdma_get_mbx_cqe_errno(cqe_status);
+               goto mbx_err;
+       }
+       if (mqe->u.rsp.status & OCRDMA_MBX_RSP_STATUS_MASK)
+               status = ocrdma_get_mbx_errno(mqe->u.rsp.status);
+mbx_err:
+       mutex_unlock(&dev->mqe_ctx.lock);
+       return status;
+}
+
+static void ocrdma_get_attr(struct ocrdma_dev *dev,
+                             struct ocrdma_dev_attr *attr,
+                             struct ocrdma_mbx_query_config *rsp)
+{
+       int max_q_mem;
+
+       attr->max_pd =
+           (rsp->max_pd_ca_ack_delay & OCRDMA_MBX_QUERY_CFG_MAX_PD_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_MAX_PD_SHIFT;
+       attr->max_qp =
+           (rsp->qp_srq_cq_ird_ord & OCRDMA_MBX_QUERY_CFG_MAX_QP_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT;
+       attr->max_send_sge = ((rsp->max_write_send_sge &
+                              OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK) >>
+                             OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT);
+       attr->max_recv_sge = (rsp->max_write_send_sge &
+                             OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT;
+       attr->max_ord_per_qp = (rsp->max_ird_ord_per_qp &
+                               OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT;
+       attr->max_ird_per_qp = (rsp->max_ird_ord_per_qp &
+                               OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_SHIFT;
+       attr->cq_overflow_detect = (rsp->qp_srq_cq_ird_ord &
+                                   OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_SHIFT;
+       attr->srq_supported = (rsp->qp_srq_cq_ird_ord &
+                              OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_SHIFT;
+       attr->local_ca_ack_delay = (rsp->max_pd_ca_ack_delay &
+                                   OCRDMA_MBX_QUERY_CFG_CA_ACK_DELAY_MASK) >>
+           OCRDMA_MBX_QUERY_CFG_CA_ACK_DELAY_SHIFT;
+       attr->max_mr = rsp->max_mr;
+       attr->max_mr_size = ~0ull;
+       attr->max_fmr = 0;
+       attr->max_pages_per_frmr = rsp->max_pages_per_frmr;
+       attr->max_num_mr_pbl = rsp->max_num_mr_pbl;
+       attr->max_cqe = rsp->max_cq_cqes_per_cq &
+                       OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_MASK;
+       attr->wqe_size = ((rsp->wqe_rqe_stride_max_dpp_cqs &
+               OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_MASK) >>
+               OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_OFFSET) *
+               OCRDMA_WQE_STRIDE;
+       attr->rqe_size = ((rsp->wqe_rqe_stride_max_dpp_cqs &
+               OCRDMA_MBX_QUERY_CFG_MAX_RQE_SIZE_MASK) >>
+               OCRDMA_MBX_QUERY_CFG_MAX_RQE_SIZE_OFFSET) *
+               OCRDMA_WQE_STRIDE;
+       attr->max_inline_data =
+           attr->wqe_size - (sizeof(struct ocrdma_hdr_wqe) +
+                             sizeof(struct ocrdma_sge));
+       max_q_mem = OCRDMA_Q_PAGE_BASE_SIZE << (OCRDMA_MAX_Q_PAGE_SIZE_CNT - 1);
+       /* hw can queue one less then the configured size,
+        * so publish less by one to stack.
+        */
+       if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
+               dev->attr.max_wqe = max_q_mem / dev->attr.wqe_size;
+               attr->ird = 1;
+               attr->ird_page_size = OCRDMA_MIN_Q_PAGE_SIZE;
+               attr->num_ird_pages = MAX_OCRDMA_IRD_PAGES;
+       } else
+               dev->attr.max_wqe = (max_q_mem / dev->attr.wqe_size) - 1;
+       dev->attr.max_rqe = (max_q_mem / dev->attr.rqe_size) - 1;
+}
+
+static int ocrdma_check_fw_config(struct ocrdma_dev *dev,
+                                  struct ocrdma_fw_conf_rsp *conf)
+{
+       u32 fn_mode;
+
+       fn_mode = conf->fn_mode & OCRDMA_FN_MODE_RDMA;
+       if (fn_mode != OCRDMA_FN_MODE_RDMA)
+               return -EINVAL;
+       dev->base_eqid = conf->base_eqid;
+       dev->max_eq = conf->max_eq;
+       dev->attr.max_cq = OCRDMA_MAX_CQ - 1;
+       return 0;
+}
+
+/* can be issued only during init time. */
+static int ocrdma_mbx_query_fw_ver(struct ocrdma_dev *dev)
+{
+       int status = -ENOMEM;
+       struct ocrdma_mqe *cmd;
+       struct ocrdma_fw_ver_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_GET_FW_VER, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       ocrdma_init_mch((struct ocrdma_mbx_hdr *)&cmd->u.cmd[0],
+                       OCRDMA_CMD_GET_FW_VER,
+                       OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_fw_ver_rsp *)cmd;
+       memset(&dev->attr.fw_ver[0], 0, sizeof(dev->attr.fw_ver));
+       memcpy(&dev->attr.fw_ver[0], &rsp->running_ver[0],
+              sizeof(rsp->running_ver));
+       ocrdma_le32_to_cpu(dev->attr.fw_ver, sizeof(rsp->running_ver));
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+/* can be issued only during init time. */
+static int ocrdma_mbx_query_fw_config(struct ocrdma_dev *dev)
+{
+       int status = -ENOMEM;
+       struct ocrdma_mqe *cmd;
+       struct ocrdma_fw_conf_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_GET_FW_CONFIG, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       ocrdma_init_mch((struct ocrdma_mbx_hdr *)&cmd->u.cmd[0],
+                       OCRDMA_CMD_GET_FW_CONFIG,
+                       OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_fw_conf_rsp *)cmd;
+       status = ocrdma_check_fw_config(dev, rsp);
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+static int ocrdma_mbx_query_dev(struct ocrdma_dev *dev)
+{
+       int status = -ENOMEM;
+       struct ocrdma_mbx_query_config *rsp;
+       struct ocrdma_mqe *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_QUERY_CONFIG, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_mbx_query_config *)cmd;
+       ocrdma_get_attr(dev, &dev->attr, rsp);
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
+{
+       int status = -ENOMEM;
+       struct ocrdma_alloc_pd *cmd;
+       struct ocrdma_alloc_pd_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_ALLOC_PD, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       if (pd->dpp_enabled)
+               cmd->enable_dpp_rsvd |= OCRDMA_ALLOC_PD_ENABLE_DPP;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_alloc_pd_rsp *)cmd;
+       pd->id = rsp->dpp_page_pdid & OCRDMA_ALLOC_PD_RSP_PDID_MASK;
+       if (rsp->dpp_page_pdid & OCRDMA_ALLOC_PD_RSP_DPP) {
+               pd->dpp_enabled = true;
+               pd->dpp_page = rsp->dpp_page_pdid >>
+                               OCRDMA_ALLOC_PD_RSP_DPP_PAGE_SHIFT;
+       } else {
+               pd->dpp_enabled = false;
+               pd->num_dpp_qp = 0;
+       }
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_dealloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
+{
+       int status = -ENOMEM;
+       struct ocrdma_dealloc_pd *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DEALLOC_PD, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->id = pd->id;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       kfree(cmd);
+       return status;
+}
+
+static int ocrdma_build_q_conf(u32 *num_entries, int entry_size,
+                              int *num_pages, int *page_size)
+{
+       int i;
+       int mem_size;
+
+       *num_entries = roundup_pow_of_two(*num_entries);
+       mem_size = *num_entries * entry_size;
+       /* find the possible lowest possible multiplier */
+       for (i = 0; i < OCRDMA_MAX_Q_PAGE_SIZE_CNT; i++) {
+               if (mem_size <= (OCRDMA_Q_PAGE_BASE_SIZE << i))
+                       break;
+       }
+       if (i >= OCRDMA_MAX_Q_PAGE_SIZE_CNT)
+               return -EINVAL;
+       mem_size = roundup(mem_size,
+                      ((OCRDMA_Q_PAGE_BASE_SIZE << i) / OCRDMA_MAX_Q_PAGES));
+       *num_pages =
+           mem_size / ((OCRDMA_Q_PAGE_BASE_SIZE << i) / OCRDMA_MAX_Q_PAGES);
+       *page_size = ((OCRDMA_Q_PAGE_BASE_SIZE << i) / OCRDMA_MAX_Q_PAGES);
+       *num_entries = mem_size / entry_size;
+       return 0;
+}
+
+static int ocrdma_mbx_create_ah_tbl(struct ocrdma_dev *dev)
+{
+       int i ;
+       int status = 0;
+       int max_ah;
+       struct ocrdma_create_ah_tbl *cmd;
+       struct ocrdma_create_ah_tbl_rsp *rsp;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       dma_addr_t pa;
+       struct ocrdma_pbe *pbes;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_AH_TBL, sizeof(*cmd));
+       if (!cmd)
+               return status;
+
+       max_ah = OCRDMA_MAX_AH;
+       dev->av_tbl.size = sizeof(struct ocrdma_av) * max_ah;
+
+       /* number of PBEs in PBL */
+       cmd->ah_conf = (OCRDMA_AH_TBL_PAGES <<
+                               OCRDMA_CREATE_AH_NUM_PAGES_SHIFT) &
+                               OCRDMA_CREATE_AH_NUM_PAGES_MASK;
+
+       /* page size */
+       for (i = 0; i < OCRDMA_MAX_Q_PAGE_SIZE_CNT; i++) {
+               if (PAGE_SIZE == (OCRDMA_MIN_Q_PAGE_SIZE << i))
+                       break;
+       }
+       cmd->ah_conf |= (i << OCRDMA_CREATE_AH_PAGE_SIZE_SHIFT) &
+                               OCRDMA_CREATE_AH_PAGE_SIZE_MASK;
+
+       /* ah_entry size */
+       cmd->ah_conf |= (sizeof(struct ocrdma_av) <<
+                               OCRDMA_CREATE_AH_ENTRY_SIZE_SHIFT) &
+                               OCRDMA_CREATE_AH_ENTRY_SIZE_MASK;
+
+       dev->av_tbl.pbl.va = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
+                                               &dev->av_tbl.pbl.pa,
+                                               GFP_KERNEL);
+       if (dev->av_tbl.pbl.va == NULL)
+               goto mem_err;
+
+       dev->av_tbl.va = dma_alloc_coherent(&pdev->dev, dev->av_tbl.size,
+                                           &pa, GFP_KERNEL);
+       if (dev->av_tbl.va == NULL)
+               goto mem_err_ah;
+       dev->av_tbl.pa = pa;
+       dev->av_tbl.num_ah = max_ah;
+       memset(dev->av_tbl.va, 0, dev->av_tbl.size);
+
+       pbes = (struct ocrdma_pbe *)dev->av_tbl.pbl.va;
+       for (i = 0; i < dev->av_tbl.size / OCRDMA_MIN_Q_PAGE_SIZE; i++) {
+               pbes[i].pa_lo = (u32) (pa & 0xffffffff);
+               pbes[i].pa_hi = (u32) upper_32_bits(pa);
+               pa += PAGE_SIZE;
+       }
+       cmd->tbl_addr[0].lo = (u32)(dev->av_tbl.pbl.pa & 0xFFFFFFFF);
+       cmd->tbl_addr[0].hi = (u32)upper_32_bits(dev->av_tbl.pbl.pa);
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_create_ah_tbl_rsp *)cmd;
+       dev->av_tbl.ahid = rsp->ahid & 0xFFFF;
+       kfree(cmd);
+       return 0;
+
+mbx_err:
+       dma_free_coherent(&pdev->dev, dev->av_tbl.size, dev->av_tbl.va,
+                         dev->av_tbl.pa);
+       dev->av_tbl.va = NULL;
+mem_err_ah:
+       dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->av_tbl.pbl.va,
+                         dev->av_tbl.pbl.pa);
+       dev->av_tbl.pbl.va = NULL;
+       dev->av_tbl.size = 0;
+mem_err:
+       kfree(cmd);
+       return status;
+}
+
+static void ocrdma_mbx_delete_ah_tbl(struct ocrdma_dev *dev)
+{
+       struct ocrdma_delete_ah_tbl *cmd;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+
+       if (dev->av_tbl.va == NULL)
+               return;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_AH_TBL, sizeof(*cmd));
+       if (!cmd)
+               return;
+       cmd->ahid = dev->av_tbl.ahid;
+
+       ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       dma_free_coherent(&pdev->dev, dev->av_tbl.size, dev->av_tbl.va,
+                         dev->av_tbl.pa);
+       dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->av_tbl.pbl.va,
+                         dev->av_tbl.pbl.pa);
+       kfree(cmd);
+}
+
+/* Multiple CQs uses the EQ. This routine returns least used
+ * EQ to associate with CQ. This will distributes the interrupt
+ * processing and CPU load to associated EQ, vector and so to that CPU.
+ */
+static u16 ocrdma_bind_eq(struct ocrdma_dev *dev)
+{
+       int i, selected_eq = 0, cq_cnt = 0;
+       u16 eq_id;
+
+       mutex_lock(&dev->dev_lock);
+       cq_cnt = dev->qp_eq_tbl[0].cq_cnt;
+       eq_id = dev->qp_eq_tbl[0].q.id;
+       /* find the EQ which is has the least number of
+        * CQs associated with it.
+        */
+       for (i = 0; i < dev->eq_cnt; i++) {
+               if (dev->qp_eq_tbl[i].cq_cnt < cq_cnt) {
+                       cq_cnt = dev->qp_eq_tbl[i].cq_cnt;
+                       eq_id = dev->qp_eq_tbl[i].q.id;
+                       selected_eq = i;
+               }
+       }
+       dev->qp_eq_tbl[selected_eq].cq_cnt += 1;
+       mutex_unlock(&dev->dev_lock);
+       return eq_id;
+}
+
+static void ocrdma_unbind_eq(struct ocrdma_dev *dev, u16 eq_id)
+{
+       int i;
+
+       mutex_lock(&dev->dev_lock);
+       for (i = 0; i < dev->eq_cnt; i++) {
+               if (dev->qp_eq_tbl[i].q.id != eq_id)
+                       continue;
+               dev->qp_eq_tbl[i].cq_cnt -= 1;
+               break;
+       }
+       mutex_unlock(&dev->dev_lock);
+}
+
+int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
+                        int entries, int dpp_cq)
+{
+       int status = -ENOMEM; int max_hw_cqe;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       struct ocrdma_create_cq *cmd;
+       struct ocrdma_create_cq_rsp *rsp;
+       u32 hw_pages, cqe_size, page_size, cqe_count;
+
+       if (dpp_cq)
+               return -EINVAL;
+       if (entries > dev->attr.max_cqe) {
+               ocrdma_err("%s(%d) max_cqe=0x%x, requester_cqe=0x%x\n",
+                          __func__, dev->id, dev->attr.max_cqe, entries);
+               return -EINVAL;
+       }
+       if (dpp_cq && (dev->nic_info.dev_family != OCRDMA_GEN2_FAMILY))
+               return -EINVAL;
+
+       if (dpp_cq) {
+               cq->max_hw_cqe = 1;
+               max_hw_cqe = 1;
+               cqe_size = OCRDMA_DPP_CQE_SIZE;
+               hw_pages = 1;
+       } else {
+               cq->max_hw_cqe = dev->attr.max_cqe;
+               max_hw_cqe = dev->attr.max_cqe;
+               cqe_size = sizeof(struct ocrdma_cqe);
+               hw_pages = OCRDMA_CREATE_CQ_MAX_PAGES;
+       }
+
+       cq->len = roundup(max_hw_cqe * cqe_size, OCRDMA_MIN_Q_PAGE_SIZE);
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_CQ, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       ocrdma_init_mch(&cmd->cmd.req, OCRDMA_CMD_CREATE_CQ,
+                       OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+       cq->va = dma_alloc_coherent(&pdev->dev, cq->len, &cq->pa, GFP_KERNEL);
+       if (!cq->va) {
+               status = -ENOMEM;
+               goto mem_err;
+       }
+       memset(cq->va, 0, cq->len);
+       page_size = cq->len / hw_pages;
+       cmd->cmd.pgsz_pgcnt = (page_size / OCRDMA_MIN_Q_PAGE_SIZE) <<
+                                       OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT;
+       cmd->cmd.pgsz_pgcnt |= hw_pages;
+       cmd->cmd.ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS;
+
+       if (dev->eq_cnt < 0)
+               goto eq_err;
+       cq->eqn = ocrdma_bind_eq(dev);
+       cmd->cmd.req.rsvd_version = OCRDMA_CREATE_CQ_VER2;
+       cqe_count = cq->len / cqe_size;
+       if (cqe_count > 1024)
+               /* Set cnt to 3 to indicate more than 1024 cq entries */
+               cmd->cmd.ev_cnt_flags |= (0x3 << OCRDMA_CREATE_CQ_CNT_SHIFT);
+       else {
+               u8 count = 0;
+               switch (cqe_count) {
+               case 256:
+                       count = 0;
+                       break;
+               case 512:
+                       count = 1;
+                       break;
+               case 1024:
+                       count = 2;
+                       break;
+               default:
+                       goto mbx_err;
+               }
+               cmd->cmd.ev_cnt_flags |= (count << OCRDMA_CREATE_CQ_CNT_SHIFT);
+       }
+       /* shared eq between all the consumer cqs. */
+       cmd->cmd.eqn = cq->eqn;
+       if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
+               if (dpp_cq)
+                       cmd->cmd.pgsz_pgcnt |= OCRDMA_CREATE_CQ_DPP <<
+                               OCRDMA_CREATE_CQ_TYPE_SHIFT;
+               cq->phase_change = false;
+               cmd->cmd.cqe_count = (cq->len / cqe_size);
+       } else {
+               cmd->cmd.cqe_count = (cq->len / cqe_size) - 1;
+               cmd->cmd.ev_cnt_flags |= OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID;
+               cq->phase_change = true;
+       }
+
+       ocrdma_build_q_pages(&cmd->cmd.pa[0], hw_pages, cq->pa, page_size);
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+
+       rsp = (struct ocrdma_create_cq_rsp *)cmd;
+       cq->id = (u16) (rsp->rsp.cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK);
+       kfree(cmd);
+       return 0;
+mbx_err:
+       ocrdma_unbind_eq(dev, cq->eqn);
+eq_err:
+       dma_free_coherent(&pdev->dev, cq->len, cq->va, cq->pa);
+mem_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq)
+{
+       int status = -ENOMEM;
+       struct ocrdma_destroy_cq *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_CQ, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       ocrdma_init_mch(&cmd->req, OCRDMA_CMD_DELETE_CQ,
+                       OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
+
+       cmd->bypass_flush_qid |=
+           (cq->id << OCRDMA_DESTROY_CQ_QID_SHIFT) &
+           OCRDMA_DESTROY_CQ_QID_MASK;
+
+       ocrdma_unbind_eq(dev, cq->eqn);
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       dma_free_coherent(&dev->nic_info.pdev->dev, cq->len, cq->va, cq->pa);
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr,
+                         u32 pdid, int addr_check)
+{
+       int status = -ENOMEM;
+       struct ocrdma_alloc_lkey *cmd;
+       struct ocrdma_alloc_lkey_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_ALLOC_LKEY, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->pdid = pdid;
+       cmd->pbl_sz_flags |= addr_check;
+       cmd->pbl_sz_flags |= (hwmr->fr_mr << OCRDMA_ALLOC_LKEY_FMR_SHIFT);
+       cmd->pbl_sz_flags |=
+           (hwmr->remote_wr << OCRDMA_ALLOC_LKEY_REMOTE_WR_SHIFT);
+       cmd->pbl_sz_flags |=
+           (hwmr->remote_rd << OCRDMA_ALLOC_LKEY_REMOTE_RD_SHIFT);
+       cmd->pbl_sz_flags |=
+           (hwmr->local_wr << OCRDMA_ALLOC_LKEY_LOCAL_WR_SHIFT);
+       cmd->pbl_sz_flags |=
+           (hwmr->remote_atomic << OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_SHIFT);
+       cmd->pbl_sz_flags |=
+           (hwmr->num_pbls << OCRDMA_ALLOC_LKEY_PBL_SIZE_SHIFT);
+
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_alloc_lkey_rsp *)cmd;
+       hwmr->lkey = rsp->lrkey;
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_dealloc_lkey(struct ocrdma_dev *dev, int fr_mr, u32 lkey)
+{
+       int status = -ENOMEM;
+       struct ocrdma_dealloc_lkey *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DEALLOC_LKEY, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       cmd->lkey = lkey;
+       cmd->rsvd_frmr = fr_mr ? 1 : 0;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+static int ocrdma_mbx_reg_mr(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr,
+                            u32 pdid, u32 pbl_cnt, u32 pbe_size, u32 last)
+{
+       int status = -ENOMEM;
+       int i;
+       struct ocrdma_reg_nsmr *cmd;
+       struct ocrdma_reg_nsmr_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_REGISTER_NSMR, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       cmd->num_pbl_pdid =
+           pdid | (hwmr->num_pbls << OCRDMA_REG_NSMR_NUM_PBL_SHIFT);
+
+       cmd->flags_hpage_pbe_sz |= (hwmr->remote_wr <<
+                                   OCRDMA_REG_NSMR_REMOTE_WR_SHIFT);
+       cmd->flags_hpage_pbe_sz |= (hwmr->remote_rd <<
+                                   OCRDMA_REG_NSMR_REMOTE_RD_SHIFT);
+       cmd->flags_hpage_pbe_sz |= (hwmr->local_wr <<
+                                   OCRDMA_REG_NSMR_LOCAL_WR_SHIFT);
+       cmd->flags_hpage_pbe_sz |= (hwmr->remote_atomic <<
+                                   OCRDMA_REG_NSMR_REMOTE_ATOMIC_SHIFT);
+       cmd->flags_hpage_pbe_sz |= (hwmr->mw_bind <<
+                                   OCRDMA_REG_NSMR_BIND_MEMWIN_SHIFT);
+       cmd->flags_hpage_pbe_sz |= (last << OCRDMA_REG_NSMR_LAST_SHIFT);
+
+       cmd->flags_hpage_pbe_sz |= (hwmr->pbe_size / OCRDMA_MIN_HPAGE_SIZE);
+       cmd->flags_hpage_pbe_sz |= (hwmr->pbl_size / OCRDMA_MIN_HPAGE_SIZE) <<
+                                       OCRDMA_REG_NSMR_HPAGE_SIZE_SHIFT;
+       cmd->totlen_low = hwmr->len;
+       cmd->totlen_high = upper_32_bits(hwmr->len);
+       cmd->fbo_low = (u32) (hwmr->fbo & 0xffffffff);
+       cmd->fbo_high = (u32) upper_32_bits(hwmr->fbo);
+       cmd->va_loaddr = (u32) hwmr->va;
+       cmd->va_hiaddr = (u32) upper_32_bits(hwmr->va);
+
+       for (i = 0; i < pbl_cnt; i++) {
+               cmd->pbl[i].lo = (u32) (hwmr->pbl_table[i].pa & 0xffffffff);
+               cmd->pbl[i].hi = upper_32_bits(hwmr->pbl_table[i].pa);
+       }
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_reg_nsmr_rsp *)cmd;
+       hwmr->lkey = rsp->lrkey;
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+static int ocrdma_mbx_reg_mr_cont(struct ocrdma_dev *dev,
+                                 struct ocrdma_hw_mr *hwmr, u32 pbl_cnt,
+                                 u32 pbl_offset, u32 last)
+{
+       int status = -ENOMEM;
+       int i;
+       struct ocrdma_reg_nsmr_cont *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_REGISTER_NSMR_CONT, sizeof(*cmd));
+       if (!cmd)
+               return -ENOMEM;
+       cmd->lrkey = hwmr->lkey;
+       cmd->num_pbl_offset = (pbl_cnt << OCRDMA_REG_NSMR_CONT_NUM_PBL_SHIFT) |
+           (pbl_offset & OCRDMA_REG_NSMR_CONT_PBL_SHIFT_MASK);
+       cmd->last = last << OCRDMA_REG_NSMR_CONT_LAST_SHIFT;
+
+       for (i = 0; i < pbl_cnt; i++) {
+               cmd->pbl[i].lo =
+                   (u32) (hwmr->pbl_table[i + pbl_offset].pa & 0xffffffff);
+               cmd->pbl[i].hi =
+                   upper_32_bits(hwmr->pbl_table[i + pbl_offset].pa);
+       }
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_reg_mr(struct ocrdma_dev *dev,
+                 struct ocrdma_hw_mr *hwmr, u32 pdid, int acc)
+{
+       int status;
+       u32 last = 0;
+       u32 cur_pbl_cnt, pbl_offset;
+       u32 pending_pbl_cnt = hwmr->num_pbls;
+
+       pbl_offset = 0;
+       cur_pbl_cnt = min(pending_pbl_cnt, MAX_OCRDMA_NSMR_PBL);
+       if (cur_pbl_cnt == pending_pbl_cnt)
+               last = 1;
+
+       status = ocrdma_mbx_reg_mr(dev, hwmr, pdid,
+                                  cur_pbl_cnt, hwmr->pbe_size, last);
+       if (status) {
+               ocrdma_err("%s() status=%d\n", __func__, status);
+               return status;
+       }
+       /* if there is no more pbls to register then exit. */
+       if (last)
+               return 0;
+
+       while (!last) {
+               pbl_offset += cur_pbl_cnt;
+               pending_pbl_cnt -= cur_pbl_cnt;
+               cur_pbl_cnt = min(pending_pbl_cnt, MAX_OCRDMA_NSMR_PBL);
+               /* if we reach the end of the pbls, then need to set the last
+                * bit, indicating no more pbls to register for this memory key.
+                */
+               if (cur_pbl_cnt == pending_pbl_cnt)
+                       last = 1;
+
+               status = ocrdma_mbx_reg_mr_cont(dev, hwmr, cur_pbl_cnt,
+                                               pbl_offset, last);
+               if (status)
+                       break;
+       }
+       if (status)
+               ocrdma_err("%s() err. status=%d\n", __func__, status);
+
+       return status;
+}
+
+bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp)
+{
+       struct ocrdma_qp *tmp;
+       bool found = false;
+       list_for_each_entry(tmp, &cq->sq_head, sq_entry) {
+               if (qp == tmp) {
+                       found = true;
+                       break;
+               }
+       }
+       return found;
+}
+
+bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp)
+{
+       struct ocrdma_qp *tmp;
+       bool found = false;
+       list_for_each_entry(tmp, &cq->rq_head, rq_entry) {
+               if (qp == tmp) {
+                       found = true;
+                       break;
+               }
+       }
+       return found;
+}
+
+void ocrdma_flush_qp(struct ocrdma_qp *qp)
+{
+       bool found;
+       unsigned long flags;
+
+       spin_lock_irqsave(&qp->dev->flush_q_lock, flags);
+       found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
+       if (!found)
+               list_add_tail(&qp->sq_entry, &qp->sq_cq->sq_head);
+       if (!qp->srq) {
+               found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
+               if (!found)
+                       list_add_tail(&qp->rq_entry, &qp->rq_cq->rq_head);
+       }
+       spin_unlock_irqrestore(&qp->dev->flush_q_lock, flags);
+}
+
+int ocrdma_qp_state_machine(struct ocrdma_qp *qp, enum ib_qp_state new_ib_state,
+                           enum ib_qp_state *old_ib_state)
+{
+       unsigned long flags;
+       int status = 0;
+       enum ocrdma_qp_state new_state;
+       new_state = get_ocrdma_qp_state(new_ib_state);
+
+       /* sync with wqe and rqe posting */
+       spin_lock_irqsave(&qp->q_lock, flags);
+
+       if (old_ib_state)
+               *old_ib_state = get_ibqp_state(qp->state);
+       if (new_state == qp->state) {
+               spin_unlock_irqrestore(&qp->q_lock, flags);
+               return 1;
+       }
+
+       switch (qp->state) {
+       case OCRDMA_QPS_RST:
+               switch (new_state) {
+               case OCRDMA_QPS_RST:
+               case OCRDMA_QPS_INIT:
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_INIT:
+               /* qps: INIT->XXX */
+               switch (new_state) {
+               case OCRDMA_QPS_INIT:
+               case OCRDMA_QPS_RTR:
+                       break;
+               case OCRDMA_QPS_ERR:
+                       ocrdma_flush_qp(qp);
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_RTR:
+               /* qps: RTS->XXX */
+               switch (new_state) {
+               case OCRDMA_QPS_RTS:
+                       break;
+               case OCRDMA_QPS_ERR:
+                       ocrdma_flush_qp(qp);
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_RTS:
+               /* qps: RTS->XXX */
+               switch (new_state) {
+               case OCRDMA_QPS_SQD:
+               case OCRDMA_QPS_SQE:
+                       break;
+               case OCRDMA_QPS_ERR:
+                       ocrdma_flush_qp(qp);
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_SQD:
+               /* qps: SQD->XXX */
+               switch (new_state) {
+               case OCRDMA_QPS_RTS:
+               case OCRDMA_QPS_SQE:
+               case OCRDMA_QPS_ERR:
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_SQE:
+               switch (new_state) {
+               case OCRDMA_QPS_RTS:
+               case OCRDMA_QPS_ERR:
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       case OCRDMA_QPS_ERR:
+               /* qps: ERR->XXX */
+               switch (new_state) {
+               case OCRDMA_QPS_RST:
+                       break;
+               default:
+                       status = -EINVAL;
+                       break;
+               };
+               break;
+       default:
+               status = -EINVAL;
+               break;
+       };
+       if (!status)
+               qp->state = new_state;
+
+       spin_unlock_irqrestore(&qp->q_lock, flags);
+       return status;
+}
+
+static u32 ocrdma_set_create_qp_mbx_access_flags(struct ocrdma_qp *qp)
+{
+       u32 flags = 0;
+       if (qp->cap_flags & OCRDMA_QP_INB_RD)
+               flags |= OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK;
+       if (qp->cap_flags & OCRDMA_QP_INB_WR)
+               flags |= OCRDMA_CREATE_QP_REQ_INB_WREN_MASK;
+       if (qp->cap_flags & OCRDMA_QP_MW_BIND)
+               flags |= OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK;
+       if (qp->cap_flags & OCRDMA_QP_LKEY0)
+               flags |= OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK;
+       if (qp->cap_flags & OCRDMA_QP_FAST_REG)
+               flags |= OCRDMA_CREATE_QP_REQ_FMR_EN_MASK;
+       return flags;
+}
+
+static int ocrdma_set_create_qp_sq_cmd(struct ocrdma_create_qp_req *cmd,
+                                       struct ib_qp_init_attr *attrs,
+                                       struct ocrdma_qp *qp)
+{
+       int status;
+       u32 len, hw_pages, hw_page_size;
+       dma_addr_t pa;
+       struct ocrdma_dev *dev = qp->dev;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       u32 max_wqe_allocated;
+       u32 max_sges = attrs->cap.max_send_sge;
+
+       max_wqe_allocated = attrs->cap.max_send_wr;
+       /* need to allocate one extra to for GEN1 family */
+       if (dev->nic_info.dev_family != OCRDMA_GEN2_FAMILY)
+               max_wqe_allocated += 1;
+
+       status = ocrdma_build_q_conf(&max_wqe_allocated,
+               dev->attr.wqe_size, &hw_pages, &hw_page_size);
+       if (status) {
+               ocrdma_err("%s() req. max_send_wr=0x%x\n", __func__,
+                          max_wqe_allocated);
+               return -EINVAL;
+       }
+       qp->sq.max_cnt = max_wqe_allocated;
+       len = (hw_pages * hw_page_size);
+
+       qp->sq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
+       if (!qp->sq.va)
+               return -EINVAL;
+       memset(qp->sq.va, 0, len);
+       qp->sq.len = len;
+       qp->sq.pa = pa;
+       qp->sq.entry_size = dev->attr.wqe_size;
+       ocrdma_build_q_pages(&cmd->wq_addr[0], hw_pages, pa, hw_page_size);
+
+       cmd->type_pgsz_pdn |= (ilog2(hw_page_size / OCRDMA_MIN_Q_PAGE_SIZE)
+                               << OCRDMA_CREATE_QP_REQ_SQ_PAGE_SIZE_SHIFT);
+       cmd->num_wq_rq_pages |= (hw_pages <<
+                                OCRDMA_CREATE_QP_REQ_NUM_WQ_PAGES_SHIFT) &
+           OCRDMA_CREATE_QP_REQ_NUM_WQ_PAGES_MASK;
+       cmd->max_sge_send_write |= (max_sges <<
+                                   OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT) &
+           OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_MASK;
+       cmd->max_sge_send_write |= (max_sges <<
+                                   OCRDMA_CREATE_QP_REQ_MAX_SGE_WRITE_SHIFT) &
+                                       OCRDMA_CREATE_QP_REQ_MAX_SGE_WRITE_MASK;
+       cmd->max_wqe_rqe |= (ilog2(qp->sq.max_cnt) <<
+                            OCRDMA_CREATE_QP_REQ_MAX_WQE_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_MAX_WQE_MASK;
+       cmd->wqe_rqe_size |= (dev->attr.wqe_size <<
+                             OCRDMA_CREATE_QP_REQ_WQE_SIZE_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_WQE_SIZE_MASK;
+       return 0;
+}
+
+static int ocrdma_set_create_qp_rq_cmd(struct ocrdma_create_qp_req *cmd,
+                                       struct ib_qp_init_attr *attrs,
+                                       struct ocrdma_qp *qp)
+{
+       int status;
+       u32 len, hw_pages, hw_page_size;
+       dma_addr_t pa = 0;
+       struct ocrdma_dev *dev = qp->dev;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       u32 max_rqe_allocated = attrs->cap.max_recv_wr + 1;
+
+       status = ocrdma_build_q_conf(&max_rqe_allocated, dev->attr.rqe_size,
+                                    &hw_pages, &hw_page_size);
+       if (status) {
+               ocrdma_err("%s() req. max_recv_wr=0x%x\n", __func__,
+                          attrs->cap.max_recv_wr + 1);
+               return status;
+       }
+       qp->rq.max_cnt = max_rqe_allocated;
+       len = (hw_pages * hw_page_size);
+
+       qp->rq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
+       if (!qp->rq.va)
+               return status;
+       memset(qp->rq.va, 0, len);
+       qp->rq.pa = pa;
+       qp->rq.len = len;
+       qp->rq.entry_size = dev->attr.rqe_size;
+
+       ocrdma_build_q_pages(&cmd->rq_addr[0], hw_pages, pa, hw_page_size);
+       cmd->type_pgsz_pdn |= (ilog2(hw_page_size / OCRDMA_MIN_Q_PAGE_SIZE) <<
+               OCRDMA_CREATE_QP_REQ_RQ_PAGE_SIZE_SHIFT);
+       cmd->num_wq_rq_pages |=
+           (hw_pages << OCRDMA_CREATE_QP_REQ_NUM_RQ_PAGES_SHIFT) &
+           OCRDMA_CREATE_QP_REQ_NUM_RQ_PAGES_MASK;
+       cmd->max_sge_recv_flags |= (attrs->cap.max_recv_sge <<
+                               OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_MASK;
+       cmd->max_wqe_rqe |= (ilog2(qp->rq.max_cnt) <<
+                               OCRDMA_CREATE_QP_REQ_MAX_RQE_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_MAX_RQE_MASK;
+       cmd->wqe_rqe_size |= (dev->attr.rqe_size <<
+                       OCRDMA_CREATE_QP_REQ_RQE_SIZE_SHIFT) &
+                       OCRDMA_CREATE_QP_REQ_RQE_SIZE_MASK;
+       return 0;
+}
+
+static void ocrdma_set_create_qp_dpp_cmd(struct ocrdma_create_qp_req *cmd,
+                                        struct ocrdma_pd *pd,
+                                        struct ocrdma_qp *qp,
+                                        u8 enable_dpp_cq, u16 dpp_cq_id)
+{
+       pd->num_dpp_qp--;
+       qp->dpp_enabled = true;
+       cmd->max_sge_recv_flags |= OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK;
+       if (!enable_dpp_cq)
+               return;
+       cmd->max_sge_recv_flags |= OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK;
+       cmd->dpp_credits_cqid = dpp_cq_id;
+       cmd->dpp_credits_cqid |= OCRDMA_CREATE_QP_REQ_DPP_CREDIT_LIMIT <<
+                                       OCRDMA_CREATE_QP_REQ_DPP_CREDIT_SHIFT;
+}
+
+static int ocrdma_set_create_qp_ird_cmd(struct ocrdma_create_qp_req *cmd,
+                                       struct ocrdma_qp *qp)
+{
+       struct ocrdma_dev *dev = qp->dev;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       dma_addr_t pa = 0;
+       int ird_page_size = dev->attr.ird_page_size;
+       int ird_q_len = dev->attr.num_ird_pages * ird_page_size;
+
+       if (dev->attr.ird == 0)
+               return 0;
+
+       qp->ird_q_va = dma_alloc_coherent(&pdev->dev, ird_q_len,
+                                       &pa, GFP_KERNEL);
+       if (!qp->ird_q_va)
+               return -ENOMEM;
+       memset(qp->ird_q_va, 0, ird_q_len);
+       ocrdma_build_q_pages(&cmd->ird_addr[0], dev->attr.num_ird_pages,
+                            pa, ird_page_size);
+       return 0;
+}
+
+static void ocrdma_get_create_qp_rsp(struct ocrdma_create_qp_rsp *rsp,
+                                    struct ocrdma_qp *qp,
+                                    struct ib_qp_init_attr *attrs,
+                                    u16 *dpp_offset, u16 *dpp_credit_lmt)
+{
+       u32 max_wqe_allocated, max_rqe_allocated;
+       qp->id = rsp->qp_id & OCRDMA_CREATE_QP_RSP_QP_ID_MASK;
+       qp->rq.dbid = rsp->sq_rq_id & OCRDMA_CREATE_QP_RSP_RQ_ID_MASK;
+       qp->sq.dbid = rsp->sq_rq_id >> OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT;
+       qp->max_ird = rsp->max_ord_ird & OCRDMA_CREATE_QP_RSP_MAX_IRD_MASK;
+       qp->max_ord = (rsp->max_ord_ird >> OCRDMA_CREATE_QP_RSP_MAX_ORD_SHIFT);
+       qp->dpp_enabled = false;
+       if (rsp->dpp_response & OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK) {
+               qp->dpp_enabled = true;
+               *dpp_credit_lmt = (rsp->dpp_response &
+                               OCRDMA_CREATE_QP_RSP_DPP_CREDITS_MASK) >>
+                               OCRDMA_CREATE_QP_RSP_DPP_CREDITS_SHIFT;
+               *dpp_offset = (rsp->dpp_response &
+                               OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_MASK) >>
+                               OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT;
+       }
+       max_wqe_allocated =
+               rsp->max_wqe_rqe >> OCRDMA_CREATE_QP_RSP_MAX_WQE_SHIFT;
+       max_wqe_allocated = 1 << max_wqe_allocated;
+       max_rqe_allocated = 1 << ((u16)rsp->max_wqe_rqe);
+
+       if (qp->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
+               qp->sq.free_delta = 0;
+               qp->rq.free_delta = 1;
+       } else
+               qp->sq.free_delta = 1;
+
+       qp->sq.max_cnt = max_wqe_allocated;
+       qp->sq.max_wqe_idx = max_wqe_allocated - 1;
+
+       if (!attrs->srq) {
+               qp->rq.max_cnt = max_rqe_allocated;
+               qp->rq.max_wqe_idx = max_rqe_allocated - 1;
+               qp->rq.free_delta = 1;
+       }
+}
+
+int ocrdma_mbx_create_qp(struct ocrdma_qp *qp, struct ib_qp_init_attr *attrs,
+                        u8 enable_dpp_cq, u16 dpp_cq_id, u16 *dpp_offset,
+                        u16 *dpp_credit_lmt)
+{
+       int status = -ENOMEM;
+       u32 flags = 0;
+       struct ocrdma_dev *dev = qp->dev;
+       struct ocrdma_pd *pd = qp->pd;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       struct ocrdma_cq *cq;
+       struct ocrdma_create_qp_req *cmd;
+       struct ocrdma_create_qp_rsp *rsp;
+       int qptype;
+
+       switch (attrs->qp_type) {
+       case IB_QPT_GSI:
+               qptype = OCRDMA_QPT_GSI;
+               break;
+       case IB_QPT_RC:
+               qptype = OCRDMA_QPT_RC;
+               break;
+       case IB_QPT_UD:
+               qptype = OCRDMA_QPT_UD;
+               break;
+       default:
+               return -EINVAL;
+       };
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_QP, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->type_pgsz_pdn |= (qptype << OCRDMA_CREATE_QP_REQ_QPT_SHIFT) &
+                                               OCRDMA_CREATE_QP_REQ_QPT_MASK;
+       status = ocrdma_set_create_qp_sq_cmd(cmd, attrs, qp);
+       if (status)
+               goto sq_err;
+
+       if (attrs->srq) {
+               struct ocrdma_srq *srq = get_ocrdma_srq(attrs->srq);
+               cmd->max_sge_recv_flags |= OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK;
+               cmd->rq_addr[0].lo = srq->id;
+               qp->srq = srq;
+       } else {
+               status = ocrdma_set_create_qp_rq_cmd(cmd, attrs, qp);
+               if (status)
+                       goto rq_err;
+       }
+
+       status = ocrdma_set_create_qp_ird_cmd(cmd, qp);
+       if (status)
+               goto mbx_err;
+
+       cmd->type_pgsz_pdn |= (pd->id << OCRDMA_CREATE_QP_REQ_PD_ID_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_PD_ID_MASK;
+
+       flags = ocrdma_set_create_qp_mbx_access_flags(qp);
+
+       cmd->max_sge_recv_flags |= flags;
+       cmd->max_ord_ird |= (dev->attr.max_ord_per_qp <<
+                            OCRDMA_CREATE_QP_REQ_MAX_ORD_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_MAX_ORD_MASK;
+       cmd->max_ord_ird |= (dev->attr.max_ird_per_qp <<
+                            OCRDMA_CREATE_QP_REQ_MAX_IRD_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_MAX_IRD_MASK;
+       cq = get_ocrdma_cq(attrs->send_cq);
+       cmd->wq_rq_cqid |= (cq->id << OCRDMA_CREATE_QP_REQ_WQ_CQID_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_WQ_CQID_MASK;
+       qp->sq_cq = cq;
+       cq = get_ocrdma_cq(attrs->recv_cq);
+       cmd->wq_rq_cqid |= (cq->id << OCRDMA_CREATE_QP_REQ_RQ_CQID_SHIFT) &
+                               OCRDMA_CREATE_QP_REQ_RQ_CQID_MASK;
+       qp->rq_cq = cq;
+
+       if (pd->dpp_enabled && attrs->cap.max_inline_data && pd->num_dpp_qp &&
+           (attrs->cap.max_inline_data <= dev->attr.max_inline_data))
+               ocrdma_set_create_qp_dpp_cmd(cmd, pd, qp, enable_dpp_cq,
+                                            dpp_cq_id);
+
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_create_qp_rsp *)cmd;
+       ocrdma_get_create_qp_rsp(rsp, qp, attrs, dpp_offset, dpp_credit_lmt);
+       qp->state = OCRDMA_QPS_RST;
+       kfree(cmd);
+       return 0;
+mbx_err:
+       if (qp->rq.va)
+               dma_free_coherent(&pdev->dev, qp->rq.len, qp->rq.va, qp->rq.pa);
+rq_err:
+       ocrdma_err("%s(%d) rq_err\n", __func__, dev->id);
+       dma_free_coherent(&pdev->dev, qp->sq.len, qp->sq.va, qp->sq.pa);
+sq_err:
+       ocrdma_err("%s(%d) sq_err\n", __func__, dev->id);
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_query_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
+                       struct ocrdma_qp_params *param)
+{
+       int status = -ENOMEM;
+       struct ocrdma_query_qp *cmd;
+       struct ocrdma_query_qp_rsp *rsp;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_QUERY_QP, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->qp_id = qp->id;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_query_qp_rsp *)cmd;
+       memcpy(param, &rsp->params, sizeof(struct ocrdma_qp_params));
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_resolve_dgid(struct ocrdma_dev *dev, union ib_gid *dgid,
+                       u8 *mac_addr)
+{
+       struct in6_addr in6;
+
+       memcpy(&in6, dgid, sizeof in6);
+       if (rdma_is_multicast_addr(&in6))
+               rdma_get_mcast_mac(&in6, mac_addr);
+       else if (rdma_link_local_addr(&in6))
+               rdma_get_ll_mac(&in6, mac_addr);
+       else {
+               ocrdma_err("%s() fail to resolve mac_addr.\n", __func__);
+               return -EINVAL;
+       }
+       return 0;
+}
+
+static void ocrdma_set_av_params(struct ocrdma_qp *qp,
+                               struct ocrdma_modify_qp *cmd,
+                               struct ib_qp_attr *attrs)
+{
+       struct ib_ah_attr *ah_attr = &attrs->ah_attr;
+       union ib_gid sgid;
+       u32 vlan_id;
+       u8 mac_addr[6];
+       if ((ah_attr->ah_flags & IB_AH_GRH) == 0)
+               return;
+       cmd->params.tclass_sq_psn |=
+           (ah_attr->grh.traffic_class << OCRDMA_QP_PARAMS_TCLASS_SHIFT);
+       cmd->params.rnt_rc_sl_fl |=
+           (ah_attr->grh.flow_label & OCRDMA_QP_PARAMS_FLOW_LABEL_MASK);
+       cmd->params.hop_lmt_rq_psn |=
+           (ah_attr->grh.hop_limit << OCRDMA_QP_PARAMS_HOP_LMT_SHIFT);
+       cmd->flags |= OCRDMA_QP_PARA_FLOW_LBL_VALID;
+       memcpy(&cmd->params.dgid[0], &ah_attr->grh.dgid.raw[0],
+              sizeof(cmd->params.dgid));
+       ocrdma_query_gid(&qp->dev->ibdev, 1,
+                        ah_attr->grh.sgid_index, &sgid);
+       qp->sgid_idx = ah_attr->grh.sgid_index;
+       memcpy(&cmd->params.sgid[0], &sgid.raw[0], sizeof(cmd->params.sgid));
+       ocrdma_resolve_dgid(qp->dev, &ah_attr->grh.dgid, &mac_addr[0]);
+       cmd->params.dmac_b0_to_b3 = mac_addr[0] | (mac_addr[1] << 8) |
+                               (mac_addr[2] << 16) | (mac_addr[3] << 24);
+       /* convert them to LE format. */
+       ocrdma_cpu_to_le32(&cmd->params.dgid[0], sizeof(cmd->params.dgid));
+       ocrdma_cpu_to_le32(&cmd->params.sgid[0], sizeof(cmd->params.sgid));
+       cmd->params.vlan_dmac_b4_to_b5 = mac_addr[4] | (mac_addr[5] << 8);
+       vlan_id = rdma_get_vlan_id(&sgid);
+       if (vlan_id && (vlan_id < 0x1000)) {
+               cmd->params.vlan_dmac_b4_to_b5 |=
+                   vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT;
+               cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID;
+       }
+}
+
+static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
+                               struct ocrdma_modify_qp *cmd,
+                               struct ib_qp_attr *attrs, int attr_mask,
+                               enum ib_qp_state old_qps)
+{
+       int status = 0;
+       struct net_device *netdev = qp->dev->nic_info.netdev;
+       int eth_mtu = iboe_get_mtu(netdev->mtu);
+
+       if (attr_mask & IB_QP_PKEY_INDEX) {
+               cmd->params.path_mtu_pkey_indx |= (attrs->pkey_index &
+                                           OCRDMA_QP_PARAMS_PKEY_INDEX_MASK);
+               cmd->flags |= OCRDMA_QP_PARA_PKEY_VALID;
+       }
+       if (attr_mask & IB_QP_QKEY) {
+               qp->qkey = attrs->qkey;
+               cmd->params.qkey = attrs->qkey;
+               cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID;
+       }
+       if (attr_mask & IB_QP_AV)
+               ocrdma_set_av_params(qp, cmd, attrs);
+       else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) {
+               /* set the default mac address for UD, GSI QPs */
+               cmd->params.dmac_b0_to_b3 = qp->dev->nic_info.mac_addr[0] |
+                       (qp->dev->nic_info.mac_addr[1] << 8) |
+                       (qp->dev->nic_info.mac_addr[2] << 16) |
+                       (qp->dev->nic_info.mac_addr[3] << 24);
+               cmd->params.vlan_dmac_b4_to_b5 = qp->dev->nic_info.mac_addr[4] |
+                                       (qp->dev->nic_info.mac_addr[5] << 8);
+       }
+       if ((attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) &&
+           attrs->en_sqd_async_notify) {
+               cmd->params.max_sge_recv_flags |=
+                       OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC;
+               cmd->flags |= OCRDMA_QP_PARA_DST_QPN_VALID;
+       }
+       if (attr_mask & IB_QP_DEST_QPN) {
+               cmd->params.ack_to_rnr_rtc_dest_qpn |= (attrs->dest_qp_num &
+                               OCRDMA_QP_PARAMS_DEST_QPN_MASK);
+               cmd->flags |= OCRDMA_QP_PARA_DST_QPN_VALID;
+       }
+       if (attr_mask & IB_QP_PATH_MTU) {
+               if (ib_mtu_enum_to_int(eth_mtu) <
+                   ib_mtu_enum_to_int(attrs->path_mtu)) {
+                       status = -EINVAL;
+                       goto pmtu_err;
+               }
+               cmd->params.path_mtu_pkey_indx |=
+                   (ib_mtu_enum_to_int(attrs->path_mtu) <<
+                    OCRDMA_QP_PARAMS_PATH_MTU_SHIFT) &
+                   OCRDMA_QP_PARAMS_PATH_MTU_MASK;
+               cmd->flags |= OCRDMA_QP_PARA_PMTU_VALID;
+       }
+       if (attr_mask & IB_QP_TIMEOUT) {
+               cmd->params.ack_to_rnr_rtc_dest_qpn |= attrs->timeout <<
+                   OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
+               cmd->flags |= OCRDMA_QP_PARA_ACK_TO_VALID;
+       }
+       if (attr_mask & IB_QP_RETRY_CNT) {
+               cmd->params.rnt_rc_sl_fl |= (attrs->retry_cnt <<
+                                     OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT) &
+                   OCRDMA_QP_PARAMS_RETRY_CNT_MASK;
+               cmd->flags |= OCRDMA_QP_PARA_RETRY_CNT_VALID;
+       }
+       if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+               cmd->params.rnt_rc_sl_fl |= (attrs->min_rnr_timer <<
+                                     OCRDMA_QP_PARAMS_RNR_NAK_TIMER_SHIFT) &
+                   OCRDMA_QP_PARAMS_RNR_NAK_TIMER_MASK;
+               cmd->flags |= OCRDMA_QP_PARA_RNT_VALID;
+       }
+       if (attr_mask & IB_QP_RNR_RETRY) {
+               cmd->params.ack_to_rnr_rtc_dest_qpn |= (attrs->rnr_retry <<
+                       OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT)
+                       & OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK;
+               cmd->flags |= OCRDMA_QP_PARA_RRC_VALID;
+       }
+       if (attr_mask & IB_QP_SQ_PSN) {
+               cmd->params.tclass_sq_psn |= (attrs->sq_psn & 0x00ffffff);
+               cmd->flags |= OCRDMA_QP_PARA_SQPSN_VALID;
+       }
+       if (attr_mask & IB_QP_RQ_PSN) {
+               cmd->params.hop_lmt_rq_psn |= (attrs->rq_psn & 0x00ffffff);
+               cmd->flags |= OCRDMA_QP_PARA_RQPSN_VALID;
+       }
+       if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+               if (attrs->max_rd_atomic > qp->dev->attr.max_ord_per_qp) {
+                       status = -EINVAL;
+                       goto pmtu_err;
+               }
+               qp->max_ord = attrs->max_rd_atomic;
+               cmd->flags |= OCRDMA_QP_PARA_MAX_ORD_VALID;
+       }
+       if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+               if (attrs->max_dest_rd_atomic > qp->dev->attr.max_ird_per_qp) {
+                       status = -EINVAL;
+                       goto pmtu_err;
+               }
+               qp->max_ird = attrs->max_dest_rd_atomic;
+               cmd->flags |= OCRDMA_QP_PARA_MAX_IRD_VALID;
+       }
+       cmd->params.max_ord_ird = (qp->max_ord <<
+                               OCRDMA_QP_PARAMS_MAX_ORD_SHIFT) |
+                               (qp->max_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK);
+pmtu_err:
+       return status;
+}
+
+int ocrdma_mbx_modify_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
+                        struct ib_qp_attr *attrs, int attr_mask,
+                        enum ib_qp_state old_qps)
+{
+       int status = -ENOMEM;
+       struct ocrdma_modify_qp *cmd;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_MODIFY_QP, sizeof(*cmd));
+       if (!cmd)
+               return status;
+
+       cmd->params.id = qp->id;
+       cmd->flags = 0;
+       if (attr_mask & IB_QP_STATE) {
+               cmd->params.max_sge_recv_flags |=
+                   (get_ocrdma_qp_state(attrs->qp_state) <<
+                    OCRDMA_QP_PARAMS_STATE_SHIFT) &
+                   OCRDMA_QP_PARAMS_STATE_MASK;
+               cmd->flags |= OCRDMA_QP_PARA_QPS_VALID;
+       } else
+               cmd->params.max_sge_recv_flags |=
+                   (qp->state << OCRDMA_QP_PARAMS_STATE_SHIFT) &
+                   OCRDMA_QP_PARAMS_STATE_MASK;
+       status = ocrdma_set_qp_params(qp, cmd, attrs, attr_mask, old_qps);
+       if (status)
+               goto mbx_err;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+
+mbx_err:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_destroy_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
+{
+       int status = -ENOMEM;
+       struct ocrdma_destroy_qp *cmd;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_QP, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->qp_id = qp->id;
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+
+mbx_err:
+       kfree(cmd);
+       if (qp->sq.va)
+               dma_free_coherent(&pdev->dev, qp->sq.len, qp->sq.va, qp->sq.pa);
+       if (!qp->srq && qp->rq.va)
+               dma_free_coherent(&pdev->dev, qp->rq.len, qp->rq.va, qp->rq.pa);
+       if (qp->dpp_enabled)
+               qp->pd->num_dpp_qp++;
+       return status;
+}
+
+int ocrdma_mbx_create_srq(struct ocrdma_srq *srq,
+                         struct ib_srq_init_attr *srq_attr,
+                         struct ocrdma_pd *pd)
+{
+       int status = -ENOMEM;
+       int hw_pages, hw_page_size;
+       int len;
+       struct ocrdma_create_srq_rsp *rsp;
+       struct ocrdma_create_srq *cmd;
+       dma_addr_t pa;
+       struct ocrdma_dev *dev = srq->dev;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       u32 max_rqe_allocated;
+
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_SRQ, sizeof(*cmd));
+       if (!cmd)
+               return status;
+
+       cmd->pgsz_pdid = pd->id & OCRDMA_CREATE_SRQ_PD_ID_MASK;
+       max_rqe_allocated = srq_attr->attr.max_wr + 1;
+       status = ocrdma_build_q_conf(&max_rqe_allocated,
+                               dev->attr.rqe_size,
+                               &hw_pages, &hw_page_size);
+       if (status) {
+               ocrdma_err("%s() req. max_wr=0x%x\n", __func__,
+                          srq_attr->attr.max_wr);
+               status = -EINVAL;
+               goto ret;
+       }
+       len = hw_pages * hw_page_size;
+       srq->rq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
+       if (!srq->rq.va) {
+               status = -ENOMEM;
+               goto ret;
+       }
+       ocrdma_build_q_pages(&cmd->rq_addr[0], hw_pages, pa, hw_page_size);
+
+       srq->rq.entry_size = dev->attr.rqe_size;
+       srq->rq.pa = pa;
+       srq->rq.len = len;
+       srq->rq.max_cnt = max_rqe_allocated;
+
+       cmd->max_sge_rqe = ilog2(max_rqe_allocated);
+       cmd->max_sge_rqe |= srq_attr->attr.max_sge <<
+                               OCRDMA_CREATE_SRQ_MAX_SGE_RECV_SHIFT;
+
+       cmd->pgsz_pdid |= (ilog2(hw_page_size / OCRDMA_MIN_Q_PAGE_SIZE)
+               << OCRDMA_CREATE_SRQ_PG_SZ_SHIFT);
+       cmd->pages_rqe_sz |= (dev->attr.rqe_size
+               << OCRDMA_CREATE_SRQ_RQE_SIZE_SHIFT)
+               & OCRDMA_CREATE_SRQ_RQE_SIZE_MASK;
+       cmd->pages_rqe_sz |= hw_pages << OCRDMA_CREATE_SRQ_NUM_RQ_PAGES_SHIFT;
+
+       status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+       if (status)
+               goto mbx_err;
+       rsp = (struct ocrdma_create_srq_rsp *)cmd;
+       srq->id = rsp->id;
+       srq->rq.dbid = rsp->id;
+       max_rqe_allocated = ((rsp->max_sge_rqe_allocated &
+               OCRDMA_CREATE_SRQ_RSP_MAX_RQE_ALLOCATED_MASK) >>
+               OCRDMA_CREATE_SRQ_RSP_MAX_RQE_ALLOCATED_SHIFT);
+       max_rqe_allocated = (1 << max_rqe_allocated);
+       srq->rq.max_cnt = max_rqe_allocated;
+       srq->rq.max_wqe_idx = max_rqe_allocated - 1;
+       srq->rq.max_sges = (rsp->max_sge_rqe_allocated &
+               OCRDMA_CREATE_SRQ_RSP_MAX_SGE_RECV_ALLOCATED_MASK) >>
+               OCRDMA_CREATE_SRQ_RSP_MAX_SGE_RECV_ALLOCATED_SHIFT;
+       goto ret;
+mbx_err:
+       dma_free_coherent(&pdev->dev, srq->rq.len, srq->rq.va, pa);
+ret:
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_modify_srq(struct ocrdma_srq *srq, struct ib_srq_attr *srq_attr)
+{
+       int status = -ENOMEM;
+       struct ocrdma_modify_srq *cmd;
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_SRQ, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->id = srq->id;
+       cmd->limit_max_rqe |= srq_attr->srq_limit <<
+           OCRDMA_MODIFY_SRQ_LIMIT_SHIFT;
+       status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd);
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_query_srq(struct ocrdma_srq *srq, struct ib_srq_attr *srq_attr)
+{
+       int status = -ENOMEM;
+       struct ocrdma_query_srq *cmd;
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_SRQ, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->id = srq->rq.dbid;
+       status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd);
+       if (status == 0) {
+               struct ocrdma_query_srq_rsp *rsp =
+                   (struct ocrdma_query_srq_rsp *)cmd;
+               srq_attr->max_sge =
+                   rsp->srq_lmt_max_sge &
+                   OCRDMA_QUERY_SRQ_RSP_MAX_SGE_RECV_MASK;
+               srq_attr->max_wr =
+                   rsp->max_rqe_pdid >> OCRDMA_QUERY_SRQ_RSP_MAX_RQE_SHIFT;
+               srq_attr->srq_limit = rsp->srq_lmt_max_sge >>
+                   OCRDMA_QUERY_SRQ_RSP_SRQ_LIMIT_SHIFT;
+       }
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_mbx_destroy_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq)
+{
+       int status = -ENOMEM;
+       struct ocrdma_destroy_srq *cmd;
+       struct pci_dev *pdev = dev->nic_info.pdev;
+       cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_SRQ, sizeof(*cmd));
+       if (!cmd)
+               return status;
+       cmd->id = srq->id;
+       status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd);
+       if (srq->rq.va)
+               dma_free_coherent(&pdev->dev, srq->rq.len,
+                                 srq->rq.va, srq->rq.pa);
+       kfree(cmd);
+       return status;
+}
+
+int ocrdma_alloc_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
+{
+       int i;
+       int status = -EINVAL;
+       struct ocrdma_av *av;
+       unsigned long flags;
+
+       av = dev->av_tbl.va;
+       spin_lock_irqsave(&dev->av_tbl.lock, flags);
+       for (i = 0; i < dev->av_tbl.num_ah; i++) {
+               if (av->valid == 0) {
+                       av->valid = OCRDMA_AV_VALID;
+                       ah->av = av;
+                       ah->id = i;
+                       status = 0;
+                       break;
+               }
+               av++;
+       }
+       if (i == dev->av_tbl.num_ah)
+               status = -EAGAIN;
+       spin_unlock_irqrestore(&dev->av_tbl.lock, flags);
+       return status;
+}
+
+int ocrdma_free_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
+{
+       unsigned long flags;
+       spin_lock_irqsave(&dev->av_tbl.lock, flags);
+       ah->av->valid = 0;
+       spin_unlock_irqrestore(&dev->av_tbl.lock, flags);
+       return 0;
+}
+
+static int ocrdma_create_mq_eq(struct ocrdma_dev *dev)
+{
+       int status;
+       int irq;
+       unsigned long flags = 0;
+       int num_eq = 0;
+
+       if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX)
+               flags = IRQF_SHARED;
+       else {
+               num_eq = dev->nic_info.msix.num_vectors -
+                               dev->nic_info.msix.start_vector;
+               /* minimum two vectors/eq are required for rdma to work.
+                * one for control path and one for data path.
+                */
+               if (num_eq < 2)
+                       return -EBUSY;
+       }
+
+       status = ocrdma_create_eq(dev, &dev->meq, OCRDMA_EQ_LEN);
+       if (status)
+               return status;
+       sprintf(dev->meq.irq_name, "ocrdma_mq%d", dev->id);
+       irq = ocrdma_get_irq(dev, &dev->meq);
+       status = request_irq(irq, ocrdma_irq_handler, flags, dev->meq.irq_name,
+                            &dev->meq);
+       if (status)
+               _ocrdma_destroy_eq(dev, &dev->meq);
+       return status;
+}
+
+static int ocrdma_create_qp_eqs(struct ocrdma_dev *dev)
+{
+       int num_eq, i, status = 0;
+       int irq;
+       unsigned long flags = 0;
+
+       num_eq = dev->nic_info.msix.num_vectors -
+                       dev->nic_info.msix.start_vector;
+       if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX) {
+               num_eq = 1;
+               flags = IRQF_SHARED;
+       } else
+               num_eq = min_t(u32, num_eq, num_online_cpus());
+       dev->qp_eq_tbl = kzalloc(sizeof(struct ocrdma_eq) * num_eq, GFP_KERNEL);
+       if (!dev->qp_eq_tbl)
+               return -ENOMEM;
+
+       for (i = 0; i < num_eq; i++) {
+               status = ocrdma_create_eq(dev, &dev->qp_eq_tbl[i],
+                                         OCRDMA_EQ_LEN);
+               if (status) {
+                       status = -EINVAL;
+                       break;
+               }
+               sprintf(dev->qp_eq_tbl[i].irq_name, "ocrdma_qp%d-%d",
+                       dev->id, i);
+               irq = ocrdma_get_irq(dev, &dev->qp_eq_tbl[i]);
+               status = request_irq(irq, ocrdma_irq_handler, flags,
+                                    dev->qp_eq_tbl[i].irq_name,
+                                    &dev->qp_eq_tbl[i]);
+               if (status) {
+                       _ocrdma_destroy_eq(dev, &dev->qp_eq_tbl[i]);
+                       status = -EINVAL;
+                       break;
+               }
+               dev->eq_cnt += 1;
+       }
+       /* one eq is sufficient for data path to work */
+       if (dev->eq_cnt >= 1)
+               return 0;
+       if (status)
+               ocrdma_destroy_qp_eqs(dev);
+       return status;
+}
+
+int ocrdma_init_hw(struct ocrdma_dev *dev)
+{
+       int status;
+       /* set up control path eq */
+       status = ocrdma_create_mq_eq(dev);
+       if (status)
+               return status;
+       /* set up data path eq */
+       status = ocrdma_create_qp_eqs(dev);
+       if (status)
+               goto qpeq_err;
+       status = ocrdma_create_mq(dev);
+       if (status)
+               goto mq_err;
+       status = ocrdma_mbx_query_fw_config(dev);
+       if (status)
+               goto conf_err;
+       status = ocrdma_mbx_query_dev(dev);
+       if (status)
+               goto conf_err;
+       status = ocrdma_mbx_query_fw_ver(dev);
+       if (status)
+               goto conf_err;
+       status = ocrdma_mbx_create_ah_tbl(dev);
+       if (status)
+               goto conf_err;
+       return 0;
+
+conf_err:
+       ocrdma_destroy_mq(dev);
+mq_err:
+       ocrdma_destroy_qp_eqs(dev);
+qpeq_err:
+       ocrdma_destroy_eq(dev, &dev->meq);
+       ocrdma_err("%s() status=%d\n", __func__, status);
+       return status;
+}
+
+void ocrdma_cleanup_hw(struct ocrdma_dev *dev)
+{
+       ocrdma_mbx_delete_ah_tbl(dev);
+
+       /* cleanup the data path eqs */
+       ocrdma_destroy_qp_eqs(dev);
+
+       /* cleanup the control path */
+       ocrdma_destroy_mq(dev);
+       ocrdma_destroy_eq(dev, &dev->meq);
+}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
new file mode 100644 (file)
index 0000000..be5db77
--- /dev/null
@@ -0,0 +1,132 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) CNA Adapters.              *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#ifndef __OCRDMA_HW_H__
+#define __OCRDMA_HW_H__
+
+#include "ocrdma_sli.h"
+
+static inline void ocrdma_cpu_to_le32(void *dst, u32 len)
+{
+#ifdef __BIG_ENDIAN
+       int i = 0;
+       u32 *src_ptr = dst;
+       u32 *dst_ptr = dst;
+       for (; i < (len / 4); i++)
+               *(dst_ptr + i) = cpu_to_le32p(src_ptr + i);
+#endif
+}
+
+static inline void ocrdma_le32_to_cpu(void *dst, u32 len)
+{
+#ifdef __BIG_ENDIAN
+       int i = 0;
+       u32 *src_ptr = dst;
+       u32 *dst_ptr = dst;
+       for (; i < (len / sizeof(u32)); i++)
+               *(dst_ptr + i) = le32_to_cpu(*(src_ptr + i));
+#endif
+}
+
+static inline void ocrdma_copy_cpu_to_le32(void *dst, void *src, u32 len)
+{
+#ifdef __BIG_ENDIAN
+       int i = 0;
+       u32 *src_ptr = src;
+       u32 *dst_ptr = dst;
+       for (; i < (len / sizeof(u32)); i++)
+               *(dst_ptr + i) = cpu_to_le32p(src_ptr + i);
+#else
+       memcpy(dst, src, len);
+#endif
+}
+
+static inline void ocrdma_copy_le32_to_cpu(void *dst, void *src, u32 len)
+{
+#ifdef __BIG_ENDIAN
+       int i = 0;
+       u32 *src_ptr = src;
+       u32 *dst_ptr = dst;
+       for (; i < len / sizeof(u32); i++)
+               *(dst_ptr + i) = le32_to_cpu(*(src_ptr + i));
+#else
+       memcpy(dst, src, len);
+#endif
+}
+
+int ocrdma_init_hw(struct ocrdma_dev *);
+void ocrdma_cleanup_hw(struct ocrdma_dev *);
+
+enum ib_qp_state get_ibqp_state(enum ocrdma_qp_state qps);
+void ocrdma_ring_cq_db(struct ocrdma_dev *, u16 cq_id, bool armed,
+                      bool solicited, u16 cqe_popped);
+
+/* verbs specific mailbox commands */
+int ocrdma_query_config(struct ocrdma_dev *,
+                       struct ocrdma_mbx_query_config *config);
+int ocrdma_resolve_dgid(struct ocrdma_dev *, union ib_gid *dgid, u8 *mac_addr);
+
+int ocrdma_mbx_alloc_pd(struct ocrdma_dev *, struct ocrdma_pd *);
+int ocrdma_mbx_dealloc_pd(struct ocrdma_dev *, struct ocrdma_pd *);
+
+int ocrdma_mbx_alloc_lkey(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr,
+                         u32 pd_id, int addr_check);
+int ocrdma_mbx_dealloc_lkey(struct ocrdma_dev *, int fmr, u32 lkey);
+
+int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr,
+                       u32 pd_id, int acc);
+int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *,
+                               int entries, int dpp_cq);
+int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *);
+
+int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs,
+                        u8 enable_dpp_cq, u16 dpp_cq_id, u16 *dpp_offset,
+                        u16 *dpp_credit_lmt);
+int ocrdma_mbx_modify_qp(struct ocrdma_dev *, struct ocrdma_qp *,
+                        struct ib_qp_attr *attrs, int attr_mask,
+                        enum ib_qp_state old_qps);
+int ocrdma_mbx_query_qp(struct ocrdma_dev *, struct ocrdma_qp *,
+                       struct ocrdma_qp_params *param);
+int ocrdma_mbx_destroy_qp(struct ocrdma_dev *, struct ocrdma_qp *);
+
+int ocrdma_mbx_create_srq(struct ocrdma_srq *,
+                         struct ib_srq_init_attr *,
+                         struct ocrdma_pd *);
+int ocrdma_mbx_modify_srq(struct ocrdma_srq *, struct ib_srq_attr *);
+int ocrdma_mbx_query_srq(struct ocrdma_srq *, struct ib_srq_attr *);
+int ocrdma_mbx_destroy_srq(struct ocrdma_dev *, struct ocrdma_srq *);
+
+int ocrdma_alloc_av(struct ocrdma_dev *, struct ocrdma_ah *);
+int ocrdma_free_av(struct ocrdma_dev *, struct ocrdma_ah *);
+
+int ocrdma_qp_state_machine(struct ocrdma_qp *, enum ib_qp_state new_state,
+                           enum ib_qp_state *old_ib_state);
+bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *);
+bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *);
+void ocrdma_flush_qp(struct ocrdma_qp *);
+
+#endif                         /* __OCRDMA_HW_H__ */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
new file mode 100644 (file)
index 0000000..a20d16e
--- /dev/null
@@ -0,0 +1,577 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/idr.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+
+#include <linux/netdevice.h>
+#include <net/addrconf.h>
+
+#include "ocrdma.h"
+#include "ocrdma_verbs.h"
+#include "ocrdma_ah.h"
+#include "be_roce.h"
+#include "ocrdma_hw.h"
+
+MODULE_VERSION(OCRDMA_ROCE_DEV_VERSION);
+MODULE_DESCRIPTION("Emulex RoCE HCA Driver");
+MODULE_AUTHOR("Emulex Corporation");
+MODULE_LICENSE("GPL");
+
+static LIST_HEAD(ocrdma_dev_list);
+static DEFINE_SPINLOCK(ocrdma_devlist_lock);
+static DEFINE_IDR(ocrdma_dev_id);
+
+static union ib_gid ocrdma_zero_sgid;
+
+static int ocrdma_get_instance(void)
+{
+       int instance = 0;
+
+       /* Assign an unused number */
+       if (!idr_pre_get(&ocrdma_dev_id, GFP_KERNEL))
+               return -1;
+       if (idr_get_new(&ocrdma_dev_id, NULL, &instance))
+               return -1;
+       return instance;
+}
+
+void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid)
+{
+       u8 mac_addr[6];
+
+       memcpy(&mac_addr[0], &dev->nic_info.mac_addr[0], ETH_ALEN);
+       guid[0] = mac_addr[0] ^ 2;
+       guid[1] = mac_addr[1];
+       guid[2] = mac_addr[2];
+       guid[3] = 0xff;
+       guid[4] = 0xfe;
+       guid[5] = mac_addr[3];
+       guid[6] = mac_addr[4];
+       guid[7] = mac_addr[5];
+}
+
+static void ocrdma_build_sgid_mac(union ib_gid *sgid, unsigned char *mac_addr,
+                                 bool is_vlan, u16 vlan_id)
+{
+       sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
+       sgid->raw[8] = mac_addr[0] ^ 2;
+       sgid->raw[9] = mac_addr[1];
+       sgid->raw[10] = mac_addr[2];
+       if (is_vlan) {
+               sgid->raw[11] = vlan_id >> 8;
+               sgid->raw[12] = vlan_id & 0xff;
+       } else {
+               sgid->raw[11] = 0xff;
+               sgid->raw[12] = 0xfe;
+       }
+       sgid->raw[13] = mac_addr[3];
+       sgid->raw[14] = mac_addr[4];
+       sgid->raw[15] = mac_addr[5];
+}
+
+static void ocrdma_add_sgid(struct ocrdma_dev *dev, unsigned char *mac_addr,
+                           bool is_vlan, u16 vlan_id)
+{
+       int i;
+       bool found = false;
+       union ib_gid new_sgid;
+       int free_idx = OCRDMA_MAX_SGID;
+       unsigned long flags;
+
+       memset(&ocrdma_zero_sgid, 0, sizeof(union ib_gid));
+
+       ocrdma_build_sgid_mac(&new_sgid, mac_addr, is_vlan, vlan_id);
+
+       spin_lock_irqsave(&dev->sgid_lock, flags);
+       for (i = 0; i < OCRDMA_MAX_SGID; i++) {
+               if (!memcmp(&dev->sgid_tbl[i], &ocrdma_zero_sgid,
+                           sizeof(union ib_gid))) {
+                       /* found free entry */
+                       if (!found) {
+                               free_idx = i;
+                               found = true;
+                               break;
+                       }
+               } else if (!memcmp(&dev->sgid_tbl[i], &new_sgid,
+                                  sizeof(union ib_gid))) {
+                       /* entry already present, no addition is required. */
+                       spin_unlock_irqrestore(&dev->sgid_lock, flags);
+                       return;
+               }
+       }
+       /* if entry doesn't exist and if table has some space, add entry */
+       if (found)
+               memcpy(&dev->sgid_tbl[free_idx], &new_sgid,
+                      sizeof(union ib_gid));
+       spin_unlock_irqrestore(&dev->sgid_lock, flags);
+}
+
+static bool ocrdma_del_sgid(struct ocrdma_dev *dev, unsigned char *mac_addr,
+                           bool is_vlan, u16 vlan_id)
+{
+       int found = false;
+       int i;
+       union ib_gid sgid;
+       unsigned long flags;
+
+       ocrdma_build_sgid_mac(&sgid, mac_addr, is_vlan, vlan_id);
+
+       spin_lock_irqsave(&dev->sgid_lock, flags);
+       /* first is default sgid, which cannot be deleted. */
+       for (i = 1; i < OCRDMA_MAX_SGID; i++) {
+               if (!memcmp(&dev->sgid_tbl[i], &sgid, sizeof(union ib_gid))) {
+                       /* found matching entry */
+                       memset(&dev->sgid_tbl[i], 0, sizeof(union ib_gid));
+                       found = true;
+                       break;
+               }
+       }
+       spin_unlock_irqrestore(&dev->sgid_lock, flags);
+       return found;
+}
+
+static void ocrdma_add_default_sgid(struct ocrdma_dev *dev)
+{
+       /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */
+       union ib_gid *sgid = &dev->sgid_tbl[0];
+
+       sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
+       ocrdma_get_guid(dev, &sgid->raw[8]);
+}
+
+static int ocrdma_build_sgid_tbl(struct ocrdma_dev *dev)
+{
+       struct net_device *netdev, *tmp;
+       u16 vlan_id;
+       bool is_vlan;
+
+       netdev = dev->nic_info.netdev;
+
+       ocrdma_add_default_sgid(dev);
+
+       rcu_read_lock();
+       for_each_netdev_rcu(&init_net, tmp) {
+               if (netdev == tmp || vlan_dev_real_dev(tmp) == netdev) {
+                       if (!netif_running(tmp) || !netif_oper_up(tmp))
+                               continue;
+                       if (netdev != tmp) {
+                               vlan_id = vlan_dev_vlan_id(tmp);
+                               is_vlan = true;
+                       } else {
+                               is_vlan = false;
+                               vlan_id = 0;
+                               tmp = netdev;
+                       }
+                       ocrdma_add_sgid(dev, tmp->dev_addr, is_vlan, vlan_id);
+               }
+       }
+       rcu_read_unlock();
+       return 0;
+}
+
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+
+static int ocrdma_inet6addr_event(struct notifier_block *notifier,
+                                 unsigned long event, void *ptr)
+{
+       struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;
+       struct net_device *event_netdev = ifa->idev->dev;
+       struct net_device *netdev = NULL;
+       struct ib_event gid_event;
+       struct ocrdma_dev *dev;
+       bool found = false;
+       bool is_vlan = false;
+       u16 vid = 0;
+
+       netdev = vlan_dev_real_dev(event_netdev);
+       if (netdev != event_netdev) {
+               is_vlan = true;
+               vid = vlan_dev_vlan_id(event_netdev);
+       }
+       rcu_read_lock();
+       list_for_each_entry_rcu(dev, &ocrdma_dev_list, entry) {
+               if (dev->nic_info.netdev == netdev) {
+                       found = true;
+                       break;
+               }
+       }
+       rcu_read_unlock();
+
+       if (!found)
+               return NOTIFY_DONE;
+       if (!rdma_link_local_addr((struct in6_addr *)&ifa->addr))
+               return NOTIFY_DONE;
+
+       mutex_lock(&dev->dev_lock);
+       switch (event) {
+       case NETDEV_UP:
+               ocrdma_add_sgid(dev, netdev->dev_addr, is_vlan, vid);
+               break;
+       case NETDEV_DOWN:
+               found = ocrdma_del_sgid(dev, netdev->dev_addr, is_vlan, vid);
+               if (found) {
+                       /* found the matching entry, notify
+                        * the consumers about it
+                        */
+                       gid_event.device = &dev->ibdev;
+                       gid_event.element.port_num = 1;
+                       gid_event.event = IB_EVENT_GID_CHANGE;
+                       ib_dispatch_event(&gid_event);
+               }
+               break;
+       default:
+               break;
+       }
+       mutex_unlock(&dev->dev_lock);
+       return NOTIFY_OK;
+}
+
+static struct notifier_block ocrdma_inet6addr_notifier = {
+       .notifier_call = ocrdma_inet6addr_event
+};
+
+#endif /* IPV6 */
+
+static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device,
+                                             u8 port_num)
+{
+       return IB_LINK_LAYER_ETHERNET;
+}
+
+static int ocrdma_register_device(struct ocrdma_dev *dev)
+{
+       strlcpy(dev->ibdev.name, "ocrdma%d", IB_DEVICE_NAME_MAX);
+       ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid);
+       memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
+              sizeof(OCRDMA_NODE_DESC));
+       dev->ibdev.owner = THIS_MODULE;
+       dev->ibdev.uverbs_cmd_mask =
+           OCRDMA_UVERBS(GET_CONTEXT) |
+           OCRDMA_UVERBS(QUERY_DEVICE) |
+           OCRDMA_UVERBS(QUERY_PORT) |
+           OCRDMA_UVERBS(ALLOC_PD) |
+           OCRDMA_UVERBS(DEALLOC_PD) |
+           OCRDMA_UVERBS(REG_MR) |
+           OCRDMA_UVERBS(DEREG_MR) |
+           OCRDMA_UVERBS(CREATE_COMP_CHANNEL) |
+           OCRDMA_UVERBS(CREATE_CQ) |
+           OCRDMA_UVERBS(RESIZE_CQ) |
+           OCRDMA_UVERBS(DESTROY_CQ) |
+           OCRDMA_UVERBS(REQ_NOTIFY_CQ) |
+           OCRDMA_UVERBS(CREATE_QP) |
+           OCRDMA_UVERBS(MODIFY_QP) |
+           OCRDMA_UVERBS(QUERY_QP) |
+           OCRDMA_UVERBS(DESTROY_QP) |
+           OCRDMA_UVERBS(POLL_CQ) |
+           OCRDMA_UVERBS(POST_SEND) |
+           OCRDMA_UVERBS(POST_RECV);
+
+       dev->ibdev.uverbs_cmd_mask |=
+           OCRDMA_UVERBS(CREATE_AH) |
+            OCRDMA_UVERBS(MODIFY_AH) |
+            OCRDMA_UVERBS(QUERY_AH) |
+            OCRDMA_UVERBS(DESTROY_AH);
+
+       dev->ibdev.node_type = RDMA_NODE_IB_CA;
+       dev->ibdev.phys_port_cnt = 1;
+       dev->ibdev.num_comp_vectors = 1;
+
+       /* mandatory verbs. */
+       dev->ibdev.query_device = ocrdma_query_device;
+       dev->ibdev.query_port = ocrdma_query_port;
+       dev->ibdev.modify_port = ocrdma_modify_port;
+       dev->ibdev.query_gid = ocrdma_query_gid;
+       dev->ibdev.get_link_layer = ocrdma_link_layer;
+       dev->ibdev.alloc_pd = ocrdma_alloc_pd;
+       dev->ibdev.dealloc_pd = ocrdma_dealloc_pd;
+
+       dev->ibdev.create_cq = ocrdma_create_cq;
+       dev->ibdev.destroy_cq = ocrdma_destroy_cq;
+       dev->ibdev.resize_cq = ocrdma_resize_cq;
+
+       dev->ibdev.create_qp = ocrdma_create_qp;
+       dev->ibdev.modify_qp = ocrdma_modify_qp;
+       dev->ibdev.query_qp = ocrdma_query_qp;
+       dev->ibdev.destroy_qp = ocrdma_destroy_qp;
+
+       dev->ibdev.query_pkey = ocrdma_query_pkey;
+       dev->ibdev.create_ah = ocrdma_create_ah;
+       dev->ibdev.destroy_ah = ocrdma_destroy_ah;
+       dev->ibdev.query_ah = ocrdma_query_ah;
+       dev->ibdev.modify_ah = ocrdma_modify_ah;
+
+       dev->ibdev.poll_cq = ocrdma_poll_cq;
+       dev->ibdev.post_send = ocrdma_post_send;
+       dev->ibdev.post_recv = ocrdma_post_recv;
+       dev->ibdev.req_notify_cq = ocrdma_arm_cq;
+
+       dev->ibdev.get_dma_mr = ocrdma_get_dma_mr;
+       dev->ibdev.dereg_mr = ocrdma_dereg_mr;
+       dev->ibdev.reg_user_mr = ocrdma_reg_user_mr;
+
+       /* mandatory to support user space verbs consumer. */
+       dev->ibdev.alloc_ucontext = ocrdma_alloc_ucontext;
+       dev->ibdev.dealloc_ucontext = ocrdma_dealloc_ucontext;
+       dev->ibdev.mmap = ocrdma_mmap;
+       dev->ibdev.dma_device = &dev->nic_info.pdev->dev;
+
+       dev->ibdev.process_mad = ocrdma_process_mad;
+
+       if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
+               dev->ibdev.uverbs_cmd_mask |=
+                    OCRDMA_UVERBS(CREATE_SRQ) |
+                    OCRDMA_UVERBS(MODIFY_SRQ) |
+                    OCRDMA_UVERBS(QUERY_SRQ) |
+                    OCRDMA_UVERBS(DESTROY_SRQ) |
+                    OCRDMA_UVERBS(POST_SRQ_RECV);
+
+               dev->ibdev.create_srq = ocrdma_create_srq;
+               dev->ibdev.modify_srq = ocrdma_modify_srq;
+               dev->ibdev.query_srq = ocrdma_query_srq;
+               dev->ibdev.destroy_srq = ocrdma_destroy_srq;
+               dev->ibdev.post_srq_recv = ocrdma_post_srq_recv;
+       }
+       return ib_register_device(&dev->ibdev, NULL);
+}
+
+static int ocrdma_alloc_resources(struct ocrdma_dev *dev)
+{
+       mutex_init(&dev->dev_lock);
+       dev->sgid_tbl = kzalloc(sizeof(union ib_gid) *
+                               OCRDMA_MAX_SGID, GFP_KERNEL);
+       if (!dev->sgid_tbl)
+               goto alloc_err;
+       spin_lock_init(&dev->sgid_lock);
+
+       dev->cq_tbl = kzalloc(sizeof(struct ocrdma_cq *) *
+                             OCRDMA_MAX_CQ, GFP_KERNEL);
+       if (!dev->cq_tbl)
+               goto alloc_err;
+
+       if (dev->attr.max_qp) {
+               dev->qp_tbl = kzalloc(sizeof(struct ocrdma_qp *) *
+                                     OCRDMA_MAX_QP, GFP_KERNEL);
+               if (!dev->qp_tbl)
+                       goto alloc_err;
+       }
+       spin_lock_init(&dev->av_tbl.lock);
+       spin_lock_init(&dev->flush_q_lock);
+       return 0;
+alloc_err:
+       ocrdma_err("%s(%d) error.\n", __func__, dev->id);
+       return -ENOMEM;
+}
+
+static void ocrdma_free_resources(struct ocrdma_dev *dev)
+{
+       kfree(dev->qp_tbl);
+       kfree(dev->cq_tbl);
+       kfree(dev->sgid_tbl);
+}
+
+static struct ocrdma_dev *ocrdma_add(struct be_dev_info *dev_info)
+{
+       int status = 0;
+       struct ocrdma_dev *dev;
+
+       dev = (struct ocrdma_dev *)ib_alloc_device(sizeof(struct ocrdma_dev));
+       if (!dev) {
+               ocrdma_err("Unable to allocate ib device\n");
+               return NULL;
+       }
+       dev->mbx_cmd = kzalloc(sizeof(struct ocrdma_mqe_emb_cmd), GFP_KERNEL);
+       if (!dev->mbx_cmd)
+               goto idr_err;
+
+       memcpy(&dev->nic_info, dev_info, sizeof(*dev_info));
+       dev->id = ocrdma_get_instance();
+       if (dev->id < 0)
+               goto idr_err;
+
+       status = ocrdma_init_hw(dev);
+       if (status)
+               goto init_err;
+
+       status = ocrdma_alloc_resources(dev);
+       if (status)
+               goto alloc_err;
+
+       status = ocrdma_build_sgid_tbl(dev);
+       if (status)
+               goto alloc_err;
+
+       status = ocrdma_register_device(dev);
+       if (status)
+               goto alloc_err;
+
+       spin_lock(&ocrdma_devlist_lock);
+       list_add_tail_rcu(&dev->entry, &ocrdma_dev_list);
+       spin_unlock(&ocrdma_devlist_lock);
+       return dev;
+
+alloc_err:
+       ocrdma_free_resources(dev);
+       ocrdma_cleanup_hw(dev);
+init_err:
+       idr_remove(&ocrdma_dev_id, dev->id);
+idr_err:
+       kfree(dev->mbx_cmd);
+       ib_dealloc_device(&dev->ibdev);
+       ocrdma_err("%s() leaving. ret=%d\n", __func__, status);
+       return NULL;
+}
+
+static void ocrdma_remove_free(struct rcu_head *rcu)
+{
+       struct ocrdma_dev *dev = container_of(rcu, struct ocrdma_dev, rcu);
+
+       ocrdma_free_resources(dev);
+       ocrdma_cleanup_hw(dev);
+
+       idr_remove(&ocrdma_dev_id, dev->id);
+       kfree(dev->mbx_cmd);
+       ib_dealloc_device(&dev->ibdev);
+}
+
+static void ocrdma_remove(struct ocrdma_dev *dev)
+{
+       /* first unregister with stack to stop all the active traffic
+        * of the registered clients.
+        */
+       ib_unregister_device(&dev->ibdev);
+
+       spin_lock(&ocrdma_devlist_lock);
+       list_del_rcu(&dev->entry);
+       spin_unlock(&ocrdma_devlist_lock);
+       call_rcu(&dev->rcu, ocrdma_remove_free);
+}
+
+static int ocrdma_open(struct ocrdma_dev *dev)
+{
+       struct ib_event port_event;
+
+       port_event.event = IB_EVENT_PORT_ACTIVE;
+       port_event.element.port_num = 1;
+       port_event.device = &dev->ibdev;
+       ib_dispatch_event(&port_event);
+       return 0;
+}
+
+static int ocrdma_close(struct ocrdma_dev *dev)
+{
+       int i;
+       struct ocrdma_qp *qp, **cur_qp;
+       struct ib_event err_event;
+       struct ib_qp_attr attrs;
+       int attr_mask = IB_QP_STATE;
+
+       attrs.qp_state = IB_QPS_ERR;
+       mutex_lock(&dev->dev_lock);
+       if (dev->qp_tbl) {
+               cur_qp = dev->qp_tbl;
+               for (i = 0; i < OCRDMA_MAX_QP; i++) {
+                       qp = cur_qp[i];
+                       if (qp) {
+                               /* change the QP state to ERROR */
+                               _ocrdma_modify_qp(&qp->ibqp, &attrs, attr_mask);
+
+                               err_event.event = IB_EVENT_QP_FATAL;
+                               err_event.element.qp = &qp->ibqp;
+                               err_event.device = &dev->ibdev;
+                               ib_dispatch_event(&err_event);
+                       }
+               }
+       }
+       mutex_unlock(&dev->dev_lock);
+
+       err_event.event = IB_EVENT_PORT_ERR;
+       err_event.element.port_num = 1;
+       err_event.device = &dev->ibdev;
+       ib_dispatch_event(&err_event);
+       return 0;
+}
+
+/* event handling via NIC driver ensures that all the NIC specific
+ * initialization done before RoCE driver notifies
+ * event to stack.
+ */
+static void ocrdma_event_handler(struct ocrdma_dev *dev, u32 event)
+{
+       switch (event) {
+       case BE_DEV_UP:
+               ocrdma_open(dev);
+               break;
+       case BE_DEV_DOWN:
+               ocrdma_close(dev);
+               break;
+       };
+}
+
+static struct ocrdma_driver ocrdma_drv = {
+       .name                   = "ocrdma_driver",
+       .add                    = ocrdma_add,
+       .remove                 = ocrdma_remove,
+       .state_change_handler   = ocrdma_event_handler,
+};
+
+static void ocrdma_unregister_inet6addr_notifier(void)
+{
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+       unregister_inet6addr_notifier(&ocrdma_inet6addr_notifier);
+#endif
+}
+
+static int __init ocrdma_init_module(void)
+{
+       int status;
+
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+       status = register_inet6addr_notifier(&ocrdma_inet6addr_notifier);
+       if (status)
+               return status;
+#endif
+
+       status = be_roce_register_driver(&ocrdma_drv);
+       if (status)
+               ocrdma_unregister_inet6addr_notifier();
+
+       return status;
+}
+
+static void __exit ocrdma_exit_module(void)
+{
+       be_roce_unregister_driver(&ocrdma_drv);
+       ocrdma_unregister_inet6addr_notifier();
+}
+
+module_init(ocrdma_init_module);
+module_exit(ocrdma_exit_module);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
new file mode 100644 (file)
index 0000000..7fd80cc
--- /dev/null
@@ -0,0 +1,1672 @@
+/*******************************************************************
+ * This file is part of the Emulex RoCE Device Driver for          *
+ * RoCE (RDMA over Converged Ethernet) adapters.                   *
+ * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
+ * EMULEX and SLI are trademarks of Emulex.                        *
+ * www.emulex.com                                                  *
+ *                                                                 *
+ * This program is free software; you can redistribute it and/or   *
+ * modify it under the terms of version 2 of the GNU General       *
+ * Public License as published by the Free Software Foundation.    *
+ * This program is distributed in the hope that it will be useful. *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
+ * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
+ * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
+ * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
+ * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
+ * more details, a copy of which can be found in the file COPYING  *
+ * included with this package.                                     *
+ *
+ * Contact Information:
+ * linux-drivers@emulex.com
+ *
+ * Emulex
+ * 3333 Susan Street
+ * Costa Mesa, CA 92626
+ *******************************************************************/
+
+#ifndef __OCRDMA_SLI_H__
+#define __OCRDMA_SLI_H__
+
+#define Bit(_b) (1 << (_b))
+
+#define OCRDMA_GEN1_FAMILY     0xB
+#define OCRDMA_GEN2_FAMILY     0x2
+
+#define OCRDMA_SUBSYS_ROCE 10
+enum {
+       OCRDMA_CMD_QUERY_CONFIG = 1,
+       OCRDMA_CMD_ALLOC_PD,
+       OCRDMA_CMD_DEALLOC_PD,
+
+       OCRDMA_CMD_CREATE_AH_TBL,
+       OCRDMA_CMD_DELETE_AH_TBL,
+
+       OCRDMA_CMD_CREATE_QP,
+       OCRDMA_CMD_QUERY_QP,
+       OCRDMA_CMD_MODIFY_QP,
+       OCRDMA_CMD_DELETE_QP,
+
+       OCRDMA_CMD_RSVD1,
+       OCRDMA_CMD_ALLOC_LKEY,
+       OCRDMA_CMD_DEALLOC_LKEY,
+       OCRDMA_CMD_REGISTER_NSMR,
+       OCRDMA_CMD_REREGISTER_NSMR,
+       OCRDMA_CMD_REGISTER_NSMR_CONT,
+       OCRDMA_CMD_QUERY_NSMR,
+       OCRDMA_CMD_ALLOC_MW,
+       OCRDMA_CMD_QUERY_MW,
+
+       OCRDMA_CMD_CREATE_SRQ,
+       OCRDMA_CMD_QUERY_SRQ,
+       OCRDMA_CMD_MODIFY_SRQ,
+       OCRDMA_CMD_DELETE_SRQ,
+
+       OCRDMA_CMD_ATTACH_MCAST,
+       OCRDMA_CMD_DETACH_MCAST,
+
+       OCRDMA_CMD_MAX
+};
+
+#define OCRDMA_SUBSYS_COMMON 1
+enum {
+       OCRDMA_CMD_CREATE_CQ            = 12,
+       OCRDMA_CMD_CREATE_EQ            = 13,
+       OCRDMA_CMD_CREATE_MQ            = 21,
+       OCRDMA_CMD_GET_FW_VER           = 35,
+       OCRDMA_CMD_DELETE_MQ            = 53,
+       OCRDMA_CMD_DELETE_CQ            = 54,
+       OCRDMA_CMD_DELETE_EQ            = 55,
+       OCRDMA_CMD_GET_FW_CONFIG        = 58,
+       OCRDMA_CMD_CREATE_MQ_EXT        = 90
+};
+
+enum {
+       QTYPE_EQ        = 1,
+       QTYPE_CQ        = 2,
+       QTYPE_MCCQ      = 3
+};
+
+#define OCRDMA_MAX_SGID (8)
+
+#define OCRDMA_MAX_QP    2048
+#define OCRDMA_MAX_CQ    2048
+
+enum {
+       OCRDMA_DB_RQ_OFFSET             = 0xE0,
+       OCRDMA_DB_GEN2_RQ1_OFFSET       = 0x100,
+       OCRDMA_DB_GEN2_RQ2_OFFSET       = 0xC0,
+       OCRDMA_DB_SQ_OFFSET             = 0x60,
+       OCRDMA_DB_GEN2_SQ_OFFSET        = 0x1C0,
+       OCRDMA_DB_SRQ_OFFSET            = OCRDMA_DB_RQ_OFFSET,
+       OCRDMA_DB_GEN2_SRQ_OFFSET       = OCRDMA_DB_GEN2_RQ1_OFFSET,
+       OCRDMA_DB_CQ_OFFSET             = 0x120,
+       OCRDMA_DB_EQ_OFFSET             = OCRDMA_DB_CQ_OFFSET,
+       OCRDMA_DB_MQ_OFFSET             = 0x140
+};
+
+#define OCRDMA_DB_CQ_RING_ID_MASK       0x3FF  /* bits 0 - 9 */
+#define OCRDMA_DB_CQ_RING_ID_EXT_MASK  0x0C00  /* bits 10-11 of qid at 12-11 */
+/* qid #2 msbits at 12-11 */
+#define OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT  0x1
+#define OCRDMA_DB_CQ_NUM_POPPED_SHIFT       (16)       /* bits 16 - 28 */
+/* Rearm bit */
+#define OCRDMA_DB_CQ_REARM_SHIFT        (29)   /* bit 29 */
+/* solicited bit */
+#define OCRDMA_DB_CQ_SOLICIT_SHIFT   (31)      /* bit 31 */
+
+#define OCRDMA_EQ_ID_MASK              0x1FF   /* bits 0 - 8 */
+#define OCRDMA_EQ_ID_EXT_MASK          0x3e00  /* bits 9-13 */
+#define OCRDMA_EQ_ID_EXT_MASK_SHIFT    (2)     /* qid bits 9-13 at 11-15 */
+
+/* Clear the interrupt for this eq */
+#define OCRDMA_EQ_CLR_SHIFT                    (9)     /* bit 9 */
+/* Must be 1 */
+#define OCRDMA_EQ_TYPE_SHIFT           (10)    /* bit 10 */
+/* Number of event entries processed */
+#define OCRDMA_NUM_EQE_SHIFT           (16)    /* bits 16 - 28 */
+/* Rearm bit */
+#define OCRDMA_REARM_SHIFT             (29)    /* bit 29 */
+
+#define OCRDMA_MQ_ID_MASK              0x7FF   /* bits 0 - 10 */
+/* Number of entries posted */
+#define OCRDMA_MQ_NUM_MQE_SHIFT        (16)    /* bits 16 - 29 */
+
+#define OCRDMA_MIN_HPAGE_SIZE (4096)
+
+#define OCRDMA_MIN_Q_PAGE_SIZE (4096)
+#define OCRDMA_MAX_Q_PAGES     (8)
+
+/*
+# 0: 4K Bytes
+# 1: 8K Bytes
+# 2: 16K Bytes
+# 3: 32K Bytes
+# 4: 64K Bytes
+*/
+#define OCRDMA_MAX_Q_PAGE_SIZE_CNT (5)
+#define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES)
+
+#define MAX_OCRDMA_QP_PAGES      (8)
+#define OCRDMA_MAX_WQE_MEM_SIZE (MAX_OCRDMA_QP_PAGES * OCRDMA_MIN_HQ_PAGE_SIZE)
+
+#define OCRDMA_CREATE_CQ_MAX_PAGES (4)
+#define OCRDMA_DPP_CQE_SIZE (4)
+
+#define OCRDMA_GEN2_MAX_CQE 1024
+#define OCRDMA_GEN2_CQ_PAGE_SIZE 4096
+#define OCRDMA_GEN2_WQE_SIZE 256
+#define OCRDMA_MAX_CQE  4095
+#define OCRDMA_CQ_PAGE_SIZE 16384
+#define OCRDMA_WQE_SIZE 128
+#define OCRDMA_WQE_STRIDE 8
+#define OCRDMA_WQE_ALIGN_BYTES 16
+
+#define MAX_OCRDMA_SRQ_PAGES MAX_OCRDMA_QP_PAGES
+
+enum {
+       OCRDMA_MCH_OPCODE_SHIFT = 0,
+       OCRDMA_MCH_OPCODE_MASK  = 0xFF,
+       OCRDMA_MCH_SUBSYS_SHIFT = 8,
+       OCRDMA_MCH_SUBSYS_MASK  = 0xFF00
+};
+
+/* mailbox cmd header */
+struct ocrdma_mbx_hdr {
+       u32 subsys_op;
+       u32 timeout;            /* in seconds */
+       u32 cmd_len;
+       u32 rsvd_version;
+} __packed;
+
+enum {
+       OCRDMA_MBX_RSP_OPCODE_SHIFT     = 0,
+       OCRDMA_MBX_RSP_OPCODE_MASK      = 0xFF,
+       OCRDMA_MBX_RSP_SUBSYS_SHIFT     = 8,
+       OCRDMA_MBX_RSP_SUBSYS_MASK      = 0xFF << OCRDMA_MBX_RSP_SUBSYS_SHIFT,
+
+       OCRDMA_MBX_RSP_STATUS_SHIFT     = 0,
+       OCRDMA_MBX_RSP_STATUS_MASK      = 0xFF,
+       OCRDMA_MBX_RSP_ASTATUS_SHIFT    = 8,
+       OCRDMA_MBX_RSP_ASTATUS_MASK     = 0xFF << OCRDMA_MBX_RSP_ASTATUS_SHIFT
+};
+
+/* mailbox cmd response */
+struct ocrdma_mbx_rsp {
+       u32 subsys_op;
+       u32 status;
+       u32 rsp_len;
+       u32 add_rsp_len;
+} __packed;
+
+enum {
+       OCRDMA_MQE_EMBEDDED     = 1,
+       OCRDMA_MQE_NONEMBEDDED  = 0
+};
+
+struct ocrdma_mqe_sge {
+       u32 pa_lo;
+       u32 pa_hi;
+       u32 len;
+} __packed;
+
+enum {
+       OCRDMA_MQE_HDR_EMB_SHIFT        = 0,
+       OCRDMA_MQE_HDR_EMB_MASK         = Bit(0),
+       OCRDMA_MQE_HDR_SGE_CNT_SHIFT    = 3,
+       OCRDMA_MQE_HDR_SGE_CNT_MASK     = 0x1F << OCRDMA_MQE_HDR_SGE_CNT_SHIFT,
+       OCRDMA_MQE_HDR_SPECIAL_SHIFT    = 24,
+       OCRDMA_MQE_HDR_SPECIAL_MASK     = 0xFF << OCRDMA_MQE_HDR_SPECIAL_SHIFT
+};
+
+struct ocrdma_mqe_hdr {
+       u32 spcl_sge_cnt_emb;
+       u32 pyld_len;
+       u32 tag_lo;
+       u32 tag_hi;
+       u32 rsvd3;
+} __packed;
+
+struct ocrdma_mqe_emb_cmd {
+       struct ocrdma_mbx_hdr mch;
+       u8 pyld[220];
+} __packed;
+
+struct ocrdma_mqe {
+       struct ocrdma_mqe_hdr hdr;
+       union {
+               struct ocrdma_mqe_emb_cmd emb_req;
+               struct {
+                       struct ocrdma_mqe_sge sge[19];
+               } nonemb_req;
+               u8 cmd[236];
+               struct ocrdma_mbx_rsp rsp;
+       } u;
+} __packed;
+
+#define OCRDMA_EQ_LEN       4096
+#define OCRDMA_MQ_CQ_LEN    256
+#define OCRDMA_MQ_LEN       128
+
+#define PAGE_SHIFT_4K          12
+#define PAGE_SIZE_4K           (1 << PAGE_SHIFT_4K)
+
+/* Returns number of pages spanned by the data starting at the given addr */
+#define PAGES_4K_SPANNED(_address, size) \
+       ((u32)((((size_t)(_address) & (PAGE_SIZE_4K - 1)) +     \
+                       (size) + (PAGE_SIZE_4K - 1)) >> PAGE_SHIFT_4K))
+
+struct ocrdma_delete_q_req {
+       struct ocrdma_mbx_hdr req;
+       u32 id;
+} __packed;
+
+struct ocrdma_pa {
+       u32 lo;
+       u32 hi;
+} __packed;
+
+#define MAX_OCRDMA_EQ_PAGES (8)
+struct ocrdma_create_eq_req {
+       struct ocrdma_mbx_hdr req;
+       u32 num_pages;
+       u32 valid;
+       u32 cnt;
+       u32 delay;
+       u32 rsvd;
+       struct ocrdma_pa pa[MAX_OCRDMA_EQ_PAGES];
+} __packed;
+
+enum {
+       OCRDMA_CREATE_EQ_VALID  = Bit(29),
+       OCRDMA_CREATE_EQ_CNT_SHIFT      = 26,
+       OCRDMA_CREATE_CQ_DELAY_SHIFT    = 13,
+};
+
+struct ocrdma_create_eq_rsp {
+       struct ocrdma_mbx_rsp rsp;
+       u32 vector_eqid;
+};
+
+#define OCRDMA_EQ_MINOR_OTHER (0x1)
+
+enum {
+       OCRDMA_MCQE_STATUS_SHIFT        = 0,
+       OCRDMA_MCQE_STATUS_MASK         = 0xFFFF,
+       OCRDMA_MCQE_ESTATUS_SHIFT       = 16,
+       OCRDMA_MCQE_ESTATUS_MASK        = 0xFFFF << OCRDMA_MCQE_ESTATUS_SHIFT,
+       OCRDMA_MCQE_CONS_SHIFT          = 27,
+       OCRDMA_MCQE_CONS_MASK           = Bit(27),
+       OCRDMA_MCQE_CMPL_SHIFT          = 28,
+       OCRDMA_MCQE_CMPL_MASK           = Bit(28),
+       OCRDMA_MCQE_AE_SHIFT            = 30,
+       OCRDMA_MCQE_AE_MASK             = Bit(30),
+       OCRDMA_MCQE_VALID_SHIFT         = 31,
+       OCRDMA_MCQE_VALID_MASK          = Bit(31)
+};
+
+struct ocrdma_mcqe {
+       u32 status;
+       u32 tag_lo;
+       u32 tag_hi;
+       u32 valid_ae_cmpl_cons;
+} __packed;
+
+enum {
+       OCRDMA_AE_MCQE_QPVALID          = Bit(31),
+       OCRDMA_AE_MCQE_QPID_MASK        = 0xFFFF,
+
+       OCRDMA_AE_MCQE_CQVALID          = Bit(31),
+       OCRDMA_AE_MCQE_CQID_MASK        = 0xFFFF,
+       OCRDMA_AE_MCQE_VALID            = Bit(31),
+       OCRDMA_AE_MCQE_AE               = Bit(30),
+       OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT = 16,
+       OCRDMA_AE_MCQE_EVENT_TYPE_MASK  =
+                                       0xFF << OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT,
+       OCRDMA_AE_MCQE_EVENT_CODE_SHIFT = 8,
+       OCRDMA_AE_MCQE_EVENT_CODE_MASK  =
+                                       0xFF << OCRDMA_AE_MCQE_EVENT_CODE_SHIFT
+};
+struct ocrdma_ae_mcqe {
+       u32 qpvalid_qpid;
+       u32 cqvalid_cqid;
+       u32 evt_tag;
+       u32 valid_ae_event;
+} __packed;
+
+enum {
+       OCRDMA_AE_MPA_MCQE_REQ_ID_SHIFT         = 16,
+       OCRDMA_AE_MPA_MCQE_REQ_ID_MASK          = 0xFFFF <<
+                                       OCRDMA_AE_MPA_MCQE_REQ_ID_SHIFT,
+
+       OCRDMA_AE_MPA_MCQE_EVENT_CODE_SHIFT     = 8,
+       OCRDMA_AE_MPA_MCQE_EVENT_CODE_MASK      = 0xFF <<
+                                       OCRDMA_AE_MPA_MCQE_EVENT_CODE_SHIFT,
+       OCRDMA_AE_MPA_MCQE_EVENT_TYPE_SHIFT     = 16,
+       OCRDMA_AE_MPA_MCQE_EVENT_TYPE_MASK      = 0xFF <<
+                                       OCRDMA_AE_MPA_MCQE_EVENT_TYPE_SHIFT,
+       OCRDMA_AE_MPA_MCQE_EVENT_AE_SHIFT       = 30,
+       OCRDMA_AE_MPA_MCQE_EVENT_AE_MASK        = Bit(30),
+       OCRDMA_AE_MPA_MCQE_EVENT_VALID_SHIFT    = 31,
+       OCRDMA_AE_MPA_MCQE_EVENT_VALID_MASK     = Bit(31)
+};
+
+struct ocrdma_ae_mpa_mcqe {
+       u32 req_id;
+       u32 w1;
+       u32 w2;
+       u32 valid_ae_event;
+} __packed;
+
+enum {
+       OCRDMA_AE_QP_MCQE_NEW_QP_STATE_SHIFT    = 0,
+       OCRDMA_AE_QP_MCQE_NEW_QP_STATE_MASK     = 0xFFFF,
+       OCRDMA_AE_QP_MCQE_QP_ID_SHIFT           = 16,
+       OCRDMA_AE_QP_MCQE_QP_ID_MASK            = 0xFFFF <<
+                                               OCRDMA_AE_QP_MCQE_QP_ID_SHIFT,
+
+       OCRDMA_AE_QP_MCQE_EVENT_CODE_SHIFT      = 8,
+       OCRDMA_AE_QP_MCQE_EVENT_CODE_MASK       = 0xFF <<
+                               OCRDMA_AE_QP_MCQE_EVENT_CODE_SHIFT,
+       OCRDMA_AE_QP_MCQE_EVENT_TYPE_SHIFT      = 16,
+       OCRDMA_AE_QP_MCQE_EVENT_TYPE_MASK       = 0xFF <<
+                               OCRDMA_AE_QP_MCQE_EVENT_TYPE_SHIFT,
+       OCRDMA_AE_QP_MCQE_EVENT_AE_SHIFT        = 30,
+       OCRDMA_AE_QP_MCQE_EVENT_AE_MASK         = Bit(30),
+       OCRDMA_AE_QP_MCQE_EVENT_VALID_SHIFT     = 31,
+       OCRDMA_AE_QP_MCQE_EVENT_VALID_MASK      = Bit(31)
+};
+
+struct ocrdma_ae_qp_mcqe {
+       u32 qp_id_state;
+       u32 w1;
+       u32 w2;
+       u32 valid_ae_event;
+} __packed;
+
+#define OCRDMA_ASYNC_EVE_CODE 0x14
+
+enum OCRDMA_ASYNC_EVENT_TYPE {
+       OCRDMA_CQ_ERROR                 = 0x00,
+       OCRDMA_CQ_OVERRUN_ERROR         = 0x01,
+       OCRDMA_CQ_QPCAT_ERROR           = 0x02,
+       OCRDMA_QP_ACCESS_ERROR          = 0x03,
+       OCRDMA_QP_COMM_EST_EVENT        = 0x04,
+       OCRDMA_SQ_DRAINED_EVENT         = 0x05,
+       OCRDMA_DEVICE_FATAL_EVENT       = 0x08,
+       OCRDMA_SRQCAT_ERROR             = 0x0E,
+       OCRDMA_SRQ_LIMIT_EVENT          = 0x0F,
+       OCRDMA_QP_LAST_WQE_EVENT        = 0x10
+};
+
+/* mailbox command request and responses */
+enum {
+       OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_SHIFT          = 2,
+       OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK           = Bit(2),
+       OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_SHIFT        = 3,
+       OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK         = Bit(3),
+       OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT               = 8,
+       OCRDMA_MBX_QUERY_CFG_MAX_QP_MASK                = 0xFFFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_PD_SHIFT               = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_PD_MASK                = 0xFFFF <<
+                                       OCRDMA_MBX_QUERY_CFG_MAX_PD_SHIFT,
+       OCRDMA_MBX_QUERY_CFG_CA_ACK_DELAY_SHIFT         = 8,
+       OCRDMA_MBX_QUERY_CFG_CA_ACK_DELAY_MASK          = 0xFF <<
+                               OCRDMA_MBX_QUERY_CFG_CA_ACK_DELAY_SHIFT,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT         = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK          = 0xFFFF,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT       = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK        = 0xFFFF,
+       OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_SHIFT       = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_MASK        = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_SHIFT,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_OFFSET        = 24,
+       OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_MASK          = 0xFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_RQE_SIZE_OFFSET        = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_RQE_SIZE_MASK          = 0xFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_RQE_SIZE_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_CQES_OFFSET        = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_CQES_MASK          = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_DPP_CQES_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_OFFSET             = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_MASK               = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_SRQ_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_RPIR_QPS_OFFSET        = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_RPIR_QPS_MASK          = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_RPIR_QPS_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_PDS_OFFSET         = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_PDS_MASK           = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_DPP_PDS_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_CREDITS_OFFSET     = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_CREDITS_MASK       = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_DPP_CREDITS_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_QPS_OFFSET         = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_DPP_QPS_MASK           = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_DPP_QPS_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_WQES_PER_WQ_OFFSET     = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_WQES_PER_WQ_MASK       = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_WQES_PER_WQ_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_RQES_PER_RQ_OFFSET     = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_RQES_PER_RQ_MASK       = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_WQES_PER_WQ_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_CQ_OFFSET              = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_CQ_MASK                = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_CQ_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_OFFSET     = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_MASK       = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_OFFSET,
+
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_RQE_OFFSET         = 16,
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_RQE_MASK           = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_SRQ_RQE_OFFSET,
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET         = 0,
+       OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_MASK           = 0xFFFF <<
+                               OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET,
+};
+
+struct ocrdma_mbx_query_config {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+       u32 qp_srq_cq_ird_ord;
+       u32 max_pd_ca_ack_delay;
+       u32 max_write_send_sge;
+       u32 max_ird_ord_per_qp;
+       u32 max_shared_ird_ord;
+       u32 max_mr;
+       u64 max_mr_size;
+       u32 max_num_mr_pbl;
+       u32 max_mw;
+       u32 max_fmr;
+       u32 max_pages_per_frmr;
+       u32 max_mcast_group;
+       u32 max_mcast_qp_attach;
+       u32 max_total_mcast_qp_attach;
+       u32 wqe_rqe_stride_max_dpp_cqs;
+       u32 max_srq_rpir_qps;
+       u32 max_dpp_pds_credits;
+       u32 max_dpp_credits_pds_per_pd;
+       u32 max_wqes_rqes_per_q;
+       u32 max_cq_cqes_per_cq;
+       u32 max_srq_rqe_sge;
+} __packed;
+
+struct ocrdma_fw_ver_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+
+       u8 running_ver[32];
+} __packed;
+
+struct ocrdma_fw_conf_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+
+       u32 config_num;
+       u32 asic_revision;
+       u32 phy_port;
+       u32 fn_mode;
+       struct {
+               u32 mode;
+               u32 nic_wqid_base;
+               u32 nic_wq_tot;
+               u32 prot_wqid_base;
+               u32 prot_wq_tot;
+               u32 prot_rqid_base;
+               u32 prot_rqid_tot;
+               u32 rsvd[6];
+       } ulp[2];
+       u32 fn_capabilities;
+       u32 rsvd1;
+       u32 rsvd2;
+       u32 base_eqid;
+       u32 max_eq;
+
+} __packed;
+
+enum {
+       OCRDMA_FN_MODE_RDMA     = 0x4
+};
+
+enum {
+       OCRDMA_CREATE_CQ_VER2                   = 2,
+
+       OCRDMA_CREATE_CQ_PAGE_CNT_MASK          = 0xFFFF,
+       OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT        = 16,
+       OCRDMA_CREATE_CQ_PAGE_SIZE_MASK         = 0xFF,
+
+       OCRDMA_CREATE_CQ_COALESCWM_SHIFT        = 12,
+       OCRDMA_CREATE_CQ_COALESCWM_MASK         = Bit(13) | Bit(12),
+       OCRDMA_CREATE_CQ_FLAGS_NODELAY          = Bit(14),
+       OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID       = Bit(15),
+
+       OCRDMA_CREATE_CQ_EQ_ID_MASK             = 0xFFFF,
+       OCRDMA_CREATE_CQ_CQE_COUNT_MASK         = 0xFFFF
+};
+
+enum {
+       OCRDMA_CREATE_CQ_VER0                   = 0,
+       OCRDMA_CREATE_CQ_DPP                    = 1,
+       OCRDMA_CREATE_CQ_TYPE_SHIFT             = 24,
+       OCRDMA_CREATE_CQ_EQID_SHIFT             = 22,
+
+       OCRDMA_CREATE_CQ_CNT_SHIFT              = 27,
+       OCRDMA_CREATE_CQ_FLAGS_VALID            = Bit(29),
+       OCRDMA_CREATE_CQ_FLAGS_EVENTABLE        = Bit(31),
+       OCRDMA_CREATE_CQ_DEF_FLAGS              = OCRDMA_CREATE_CQ_FLAGS_VALID |
+                                       OCRDMA_CREATE_CQ_FLAGS_EVENTABLE |
+                                       OCRDMA_CREATE_CQ_FLAGS_NODELAY
+};
+
+struct ocrdma_create_cq_cmd {
+       struct ocrdma_mbx_hdr req;
+       u32 pgsz_pgcnt;
+       u32 ev_cnt_flags;
+       u32 eqn;
+       u32 cqe_count;
+       u32 rsvd6;
+       struct ocrdma_pa pa[OCRDMA_CREATE_CQ_MAX_PAGES];
+};
+
+struct ocrdma_create_cq {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_create_cq_cmd cmd;
+} __packed;
+
+enum {
+       OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK = 0xFFFF
+};
+
+struct ocrdma_create_cq_cmd_rsp {
+       struct ocrdma_mbx_rsp rsp;
+       u32 cq_id;
+} __packed;
+
+struct ocrdma_create_cq_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_create_cq_cmd_rsp rsp;
+} __packed;
+
+enum {
+       OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT         = 22,
+       OCRDMA_CREATE_MQ_CQ_ID_SHIFT            = 16,
+       OCRDMA_CREATE_MQ_RING_SIZE_SHIFT        = 16,
+       OCRDMA_CREATE_MQ_VALID                  = Bit(31),
+       OCRDMA_CREATE_MQ_ASYNC_CQ_VALID         = Bit(0)
+};
+
+struct ocrdma_create_mq_v0 {
+       u32 pages;
+       u32 cqid_ringsize;
+       u32 valid;
+       u32 async_cqid_valid;
+       u32 rsvd;
+       struct ocrdma_pa pa[8];
+} __packed;
+
+struct ocrdma_create_mq_v1 {
+       u32 cqid_pages;
+       u32 async_event_bitmap;
+       u32 async_cqid_ringsize;
+       u32 valid;
+       u32 async_cqid_valid;
+       u32 rsvd;
+       struct ocrdma_pa pa[8];
+} __packed;
+
+struct ocrdma_create_mq_req {
+       struct ocrdma_mbx_hdr req;
+       union {
+               struct ocrdma_create_mq_v0 v0;
+               struct ocrdma_create_mq_v1 v1;
+       };
+} __packed;
+
+struct ocrdma_create_mq_rsp {
+       struct ocrdma_mbx_rsp rsp;
+       u32 id;
+} __packed;
+
+enum {
+       OCRDMA_DESTROY_CQ_QID_SHIFT                     = 0,
+       OCRDMA_DESTROY_CQ_QID_MASK                      = 0xFFFF,
+       OCRDMA_DESTROY_CQ_QID_BYPASS_FLUSH_SHIFT        = 16,
+       OCRDMA_DESTROY_CQ_QID_BYPASS_FLUSH_MASK         = 0xFFFF <<
+                               OCRDMA_DESTROY_CQ_QID_BYPASS_FLUSH_SHIFT
+};
+
+struct ocrdma_destroy_cq {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_hdr req;
+
+       u32 bypass_flush_qid;
+} __packed;
+
+struct ocrdma_destroy_cq_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+} __packed;
+
+enum {
+       OCRDMA_QPT_GSI  = 1,
+       OCRDMA_QPT_RC   = 2,
+       OCRDMA_QPT_UD   = 4,
+};
+
+enum {
+       OCRDMA_CREATE_QP_REQ_PD_ID_SHIFT        = 0,
+       OCRDMA_CREATE_QP_REQ_PD_ID_MASK         = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_SQ_PAGE_SIZE_SHIFT = 16,
+       OCRDMA_CREATE_QP_REQ_RQ_PAGE_SIZE_SHIFT = 19,
+       OCRDMA_CREATE_QP_REQ_QPT_SHIFT          = 29,
+       OCRDMA_CREATE_QP_REQ_QPT_MASK           = Bit(31) | Bit(30) | Bit(29),
+
+       OCRDMA_CREATE_QP_REQ_MAX_RQE_SHIFT      = 0,
+       OCRDMA_CREATE_QP_REQ_MAX_RQE_MASK       = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_MAX_WQE_SHIFT      = 16,
+       OCRDMA_CREATE_QP_REQ_MAX_WQE_MASK       = 0xFFFF <<
+                                       OCRDMA_CREATE_QP_REQ_MAX_WQE_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_WRITE_SHIFT        = 0,
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_WRITE_MASK         = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT         = 16,
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_MASK          = 0xFFFF <<
+                                       OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_FMR_EN_SHIFT               = 0,
+       OCRDMA_CREATE_QP_REQ_FMR_EN_MASK                = Bit(0),
+       OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_SHIFT          = 1,
+       OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK           = Bit(1),
+       OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_SHIFT          = 2,
+       OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK           = Bit(2),
+       OCRDMA_CREATE_QP_REQ_INB_WREN_SHIFT             = 3,
+       OCRDMA_CREATE_QP_REQ_INB_WREN_MASK              = Bit(3),
+       OCRDMA_CREATE_QP_REQ_INB_RDEN_SHIFT             = 4,
+       OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK              = Bit(4),
+       OCRDMA_CREATE_QP_REQ_USE_SRQ_SHIFT              = 5,
+       OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK               = Bit(5),
+       OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_SHIFT          = 6,
+       OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_MASK           = Bit(6),
+       OCRDMA_CREATE_QP_REQ_ENABLE_DPP_SHIFT           = 7,
+       OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK            = Bit(7),
+       OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_SHIFT        = 8,
+       OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_MASK         = Bit(8),
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT         = 16,
+       OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_MASK          = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_MAX_IRD_SHIFT              = 0,
+       OCRDMA_CREATE_QP_REQ_MAX_IRD_MASK               = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_MAX_ORD_SHIFT              = 16,
+       OCRDMA_CREATE_QP_REQ_MAX_ORD_MASK               = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_MAX_ORD_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_NUM_RQ_PAGES_SHIFT         = 0,
+       OCRDMA_CREATE_QP_REQ_NUM_RQ_PAGES_MASK          = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_NUM_WQ_PAGES_SHIFT         = 16,
+       OCRDMA_CREATE_QP_REQ_NUM_WQ_PAGES_MASK          = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_NUM_WQ_PAGES_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_RQE_SIZE_SHIFT             = 0,
+       OCRDMA_CREATE_QP_REQ_RQE_SIZE_MASK              = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_WQE_SIZE_SHIFT             = 16,
+       OCRDMA_CREATE_QP_REQ_WQE_SIZE_MASK              = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_WQE_SIZE_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_RQ_CQID_SHIFT              = 0,
+       OCRDMA_CREATE_QP_REQ_RQ_CQID_MASK               = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_WQ_CQID_SHIFT              = 16,
+       OCRDMA_CREATE_QP_REQ_WQ_CQID_MASK               = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_WQ_CQID_SHIFT,
+
+       OCRDMA_CREATE_QP_REQ_DPP_CQPID_SHIFT            = 0,
+       OCRDMA_CREATE_QP_REQ_DPP_CQPID_MASK             = 0xFFFF,
+       OCRDMA_CREATE_QP_REQ_DPP_CREDIT_SHIFT           = 16,
+       OCRDMA_CREATE_QP_REQ_DPP_CREDIT_MASK            = 0xFFFF <<
+                               OCRDMA_CREATE_QP_REQ_DPP_CREDIT_SHIFT
+};
+
+enum {
+       OCRDMA_CREATE_QP_REQ_DPP_CREDIT_LIMIT   = 16,
+       OCRDMA_CREATE_QP_RSP_DPP_PAGE_SHIFT     = 1
+};
+
+#define MAX_OCRDMA_IRD_PAGES 4
+
+enum ocrdma_qp_flags {
+       OCRDMA_QP_MW_BIND       = 1,
+       OCRDMA_QP_LKEY0         = (1 << 1),
+       OCRDMA_QP_FAST_REG      = (1 << 2),
+       OCRDMA_QP_INB_RD        = (1 << 6),
+       OCRDMA_QP_INB_WR        = (1 << 7),
+};
+
+enum ocrdma_qp_state {
+       OCRDMA_QPS_RST          = 0,
+       OCRDMA_QPS_INIT         = 1,
+       OCRDMA_QPS_RTR          = 2,
+       OCRDMA_QPS_RTS          = 3,
+       OCRDMA_QPS_SQE          = 4,
+       OCRDMA_QPS_SQ_DRAINING  = 5,
+       OCRDMA_QPS_ERR          = 6,
+       OCRDMA_QPS_SQD          = 7
+};
+
+struct ocrdma_create_qp_req {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_hdr req;
+
+       u32 type_pgsz_pdn;
+       u32 max_wqe_rqe;
+       u32 max_sge_send_write;
+       u32 max_sge_recv_flags;
+       u32 max_ord_ird;
+       u32 num_wq_rq_pages;
+       u32 wqe_rqe_size;
+       u32 wq_rq_cqid;
+       struct ocrdma_pa wq_addr[MAX_OCRDMA_QP_PAGES];
+       struct ocrdma_pa rq_addr[MAX_OCRDMA_QP_PAGES];
+       u32 dpp_credits_cqid;
+       u32 rpir_lkey;
+       struct ocrdma_pa ird_addr[MAX_OCRDMA_IRD_PAGES];
+} __packed;
+
+enum {
+       OCRDMA_CREATE_QP_RSP_QP_ID_SHIFT                = 0,
+       OCRDMA_CREATE_QP_RSP_QP_ID_MASK                 = 0xFFFF,
+
+       OCRDMA_CREATE_QP_RSP_MAX_RQE_SHIFT              = 0,
+       OCRDMA_CREATE_QP_RSP_MAX_RQE_MASK               = 0xFFFF,
+       OCRDMA_CREATE_QP_RSP_MAX_WQE_SHIFT              = 16,
+       OCRDMA_CREATE_QP_RSP_MAX_WQE_MASK               = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_MAX_WQE_SHIFT,
+
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_WRITE_SHIFT        = 0,
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_WRITE_MASK         = 0xFFFF,
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_SEND_SHIFT         = 16,
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_SEND_MASK          = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_MAX_SGE_SEND_SHIFT,
+
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_RECV_SHIFT         = 16,
+       OCRDMA_CREATE_QP_RSP_MAX_SGE_RECV_MASK          = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_MAX_SGE_RECV_SHIFT,
+
+       OCRDMA_CREATE_QP_RSP_MAX_IRD_SHIFT              = 0,
+       OCRDMA_CREATE_QP_RSP_MAX_IRD_MASK               = 0xFFFF,
+       OCRDMA_CREATE_QP_RSP_MAX_ORD_SHIFT              = 16,
+       OCRDMA_CREATE_QP_RSP_MAX_ORD_MASK               = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_MAX_ORD_SHIFT,
+
+       OCRDMA_CREATE_QP_RSP_RQ_ID_SHIFT                = 0,
+       OCRDMA_CREATE_QP_RSP_RQ_ID_MASK                 = 0xFFFF,
+       OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT                = 16,
+       OCRDMA_CREATE_QP_RSP_SQ_ID_MASK                 = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT,
+
+       OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK           = Bit(0),
+       OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT      = 1,
+       OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_MASK       = 0x7FFF <<
+                               OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT,
+       OCRDMA_CREATE_QP_RSP_DPP_CREDITS_SHIFT          = 16,
+       OCRDMA_CREATE_QP_RSP_DPP_CREDITS_MASK           = 0xFFFF <<
+                               OCRDMA_CREATE_QP_RSP_DPP_CREDITS_SHIFT,
+};
+
+struct ocrdma_create_qp_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+
+       u32 qp_id;
+       u32 max_wqe_rqe;
+       u32 max_sge_send_write;
+       u32 max_sge_recv;
+       u32 max_ord_ird;
+       u32 sq_rq_id;
+       u32 dpp_response;
+} __packed;
+
+struct ocrdma_destroy_qp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_hdr req;
+       u32 qp_id;
+} __packed;
+
+struct ocrdma_destroy_qp_rsp {
+       struct ocrdma_mqe_hdr hdr;
+       struct ocrdma_mbx_rsp rsp;
+} __packed;
+
+enum {
+       OCRDMA_MODIFY_QP_ID_SHIFT       = 0,
+       OCRDMA_MODIFY_QP_ID_MASK        = 0xFFFF,
+
+       OCRDMA_QP_PARA_QPS_VALID        = Bit(0),
+       OCRDMA_QP_PARA_SQD_ASYNC_VALID  = Bit(1),
+       OCRDMA_QP_PARA_PKEY_VALID       = Bit(2),
+       OCRDMA_QP_PARA_QKEY_VALID       = Bit(3),
+       OCRDMA_QP_PARA_PMTU_VALID       = Bit(4),
+       OCRDMA_QP_PARA_ACK_TO_VALID     = Bit(5),
+       OCRDMA_QP_PARA_RETRY_CNT_VALID  = Bit(6),
+       OCRDMA_QP_PARA_RRC_VALID        = Bit(7),
+       OCRDMA_QP_PARA_RQPSN_VALID      = Bit(8),
+       OCRDMA_QP_PARA_MAX_IRD_VALID    = Bit(9),
+       OCRDMA_QP_PARA_MAX_ORD_VALID    = Bit(10),
+       OCRDMA_QP_PARA_RNT_VALID        = Bit(11),
+       OCRDMA_QP_PARA_SQPSN_VALID      = Bit(12),
+       OCRDMA_QP_PARA_DST_QPN_VALID    = Bit(13),
+       OCRDMA_QP_PARA_MAX_WQE_VALID    = Bit(14),
+       OCRDMA_QP_PARA_MAX_RQE_VALID    = Bit(15),
+       OCRDMA_QP_PARA_SGE_SEND_VALID   = Bit(16),
+       OCRDMA_QP_PARA_SGE_RECV_VALID   = Bit(17),
+       OCRDMA_QP_PARA_SGE_WR_VALID     = Bit(18),
+       OCRDMA_QP_PARA_INB_RDEN_VALID   = Bit(19),
+       OCRDMA_QP_PARA_INB_WREN_VALID   = Bit(20),
+       OCRDMA_QP_PARA_FLOW_LBL_VALID   = Bit(21),
+       OCRDMA_QP_PARA_BIND_EN_VALID    = Bit(22),
+       OCRDMA_QP_PARA_ZLKEY_EN_VALID   = Bit(23),
+       OCRDMA_QP_PARA_FMR_EN_VALID     = Bit(24),
+       OCRDMA_QP_PARA_INBAT_EN_VALID   = Bit(25),
+       OCRDMA_QP_PARA_VLAN_EN_VALID    = Bit(26),
+
+       OCRDMA_MODIFY_QP_FLAGS_RD       = Bit(0),
+       OCRDMA_MODIFY_QP_FLAGS_WR       = Bit(1),
+       OCRDMA_MODIFY_QP_FLAGS_SEND     = Bit(2),
+       OCRDMA_MODIFY_QP_FLAGS_ATOMIC   = Bit(3)
+};
+
+enum {
+       OCRDMA_QP_PARAMS_SRQ_ID_SHIFT           = 0,
+       OCRDMA_QP_PARAMS_SRQ_ID_MASK            = 0xFFFF,
+
+       OCRDMA_QP_PARAMS_MAX_RQE_SHIFT          = 0,
+       OCRDMA_QP_PARAMS_MAX_RQE_MASK           = 0xFFFF,
+       OCRDMA_QP_PARAMS_MAX_WQE_SHIFT          = 16,
+       OCRDMA_QP_PARAMS_MAX_WQE_MASK           = 0xFFFF <<
+           OCRDMA_QP_PARAMS_MAX_WQE_SHIFT,
+
+       OCRDMA_QP_PARAMS_MAX_SGE_WRITE_SHIFT    = 0,
+       OCRDMA_QP_PARAMS_MAX_SGE_WRITE_MASK     = 0xFFFF,
+       OCRDMA_QP_PARAMS_MAX_SGE_SEND_SHIFT     = 16,
+       OCRDMA_QP_PARAMS_MAX_SGE_SEND_MASK      = 0xFFFF <<
+                                       OCRDMA_QP_PARAMS_MAX_SGE_SEND_SHIFT,
+
+       OCRDMA_QP_PARAMS_FLAGS_FMR_EN           = Bit(0),
+       OCRDMA_QP_PARAMS_FLAGS_LKEY_0_EN        = Bit(1),
+       OCRDMA_QP_PARAMS_FLAGS_BIND_MW_EN       = Bit(2),
+       OCRDMA_QP_PARAMS_FLAGS_INBWR_EN         = Bit(3),
+       OCRDMA_QP_PARAMS_FLAGS_INBRD_EN         = Bit(4),
+       OCRDMA_QP_PARAMS_STATE_SHIFT            = 5,
+       OCRDMA_QP_PARAMS_STATE_MASK             = Bit(5) | Bit(6) | Bit(7),
+       OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC        = Bit(8),
+       OCRDMA_QP_PARAMS_FLAGS_INB_ATEN         = Bit(9),
+       OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT     = 16,
+       OCRDMA_QP_PARAMS_MAX_SGE_RECV_MASK      = 0xFFFF <<
+                                       OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT,
+
+       OCRDMA_QP_PARAMS_MAX_IRD_SHIFT          = 0,
+       OCRDMA_QP_PARAMS_MAX_IRD_MASK           = 0xFFFF,
+       OCRDMA_QP_PARAMS_MAX_ORD_SHIFT          = 16,
+       OCRDMA_QP_PARAMS_MAX_ORD_MASK           = 0xFFFF <<
+                                       OCRDMA_QP_PARAMS_MAX_ORD_SHIFT,
+
+       OCRDMA_QP_PARAMS_RQ_CQID_SHIFT          = 0,
+       OCRDMA_QP_PARAMS_RQ_CQID_MASK           = 0xFFFF,
+       OCRDMA_QP_PARAMS_WQ_CQID_SHIFT          = 16,
+       OCRDMA_QP_PARAMS_WQ_CQID_MASK           = 0xFFFF <<
+                                       OCRDMA_QP_PARAMS_WQ_CQID_SHIFT,
+
+       OCRDMA_QP_PARAMS_RQ_PSN_SHIFT           = 0,
+       OCRDMA_QP_PARAMS_RQ_PSN_MASK            = 0xFFFFFF,
+       OCRDMA_QP_PARAMS_HOP_LMT_SHIFT          = 24,
+       OCRDMA_QP_PARAMS_HOP_LMT_MASK           = 0xFF <<
+                                       OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,
+
+       OCRDMA_QP_PARAMS_SQ_PSN_SHIFT           = 0,
+       OCRDMA_QP_PARAMS_SQ_PSN_MASK            = 0xFFFFFF,
+       OCRDMA_QP_PARAMS_TCLASS_SHIFT           = 24,
+       OCRDMA_QP_PARAMS_TCLASS_MASK            = 0xFF <<
+                                       OCRDMA_QP_PARAMS_TCLASS_SHIFT,
+
+       OCRDMA_QP_PARAMS_DEST_QPN_SHIFT         = 0,
+       OCRDMA_QP_PARAMS_DEST_QPN_MASK          = 0xFFFFFF,
+       OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT    = 24,
+ &nb