RDS/IB: Use SLAB_HWCACHE_ALIGN flag for kmem_cache_create()
[linux-2.6.git] / net / rds / ib_recv.c
index 5b429b7..37dab28 100644 (file)
@@ -238,7 +238,8 @@ void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
                rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
 }
 
-static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic)
+static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,
+                                                    gfp_t slab_mask)
 {
        struct rds_ib_incoming *ibinc;
        struct list_head *cache_item;
@@ -254,7 +255,7 @@ static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *i
                        rds_ib_stats_inc(s_ib_rx_alloc_limit);
                        return NULL;
                }
-               ibinc = kmem_cache_alloc(rds_ib_incoming_slab, GFP_NOWAIT);
+               ibinc = kmem_cache_alloc(rds_ib_incoming_slab, slab_mask);
                if (!ibinc) {
                        atomic_dec(&rds_ib_allocation);
                        return NULL;
@@ -266,7 +267,8 @@ static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *i
        return ibinc;
 }
 
-static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic)
+static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,
+                                                   gfp_t slab_mask, gfp_t page_mask)
 {
        struct rds_page_frag *frag;
        struct list_head *cache_item;
@@ -276,12 +278,12 @@ static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic
        if (cache_item) {
                frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
        } else {
-               frag = kmem_cache_alloc(rds_ib_frag_slab, GFP_NOWAIT);
+               frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
                if (!frag)
                        return NULL;
 
                ret = rds_page_remainder_alloc(&frag->f_sg,
-                                              RDS_FRAG_SIZE, GFP_NOWAIT);
+                                              RDS_FRAG_SIZE, page_mask);
                if (ret) {
                        kmem_cache_free(rds_ib_frag_slab, frag);
                        return NULL;
@@ -294,11 +296,18 @@ static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic
 }
 
 static int rds_ib_recv_refill_one(struct rds_connection *conn,
-                                 struct rds_ib_recv_work *recv)
+                                 struct rds_ib_recv_work *recv, int prefill)
 {
        struct rds_ib_connection *ic = conn->c_transport_data;
        struct ib_sge *sge;
        int ret = -ENOMEM;
+       gfp_t slab_mask = GFP_NOWAIT;
+       gfp_t page_mask = GFP_NOWAIT;
+
+       if (prefill) {
+               slab_mask = GFP_KERNEL;
+               page_mask = GFP_HIGHUSER;
+       }
 
        if (!ic->i_cache_incs.ready)
                rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
@@ -310,13 +319,13 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
         * recvs that were continuations will still have this allocated.
         */
        if (!recv->r_ibinc) {
-               recv->r_ibinc = rds_ib_refill_one_inc(ic);
+               recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);
                if (!recv->r_ibinc)
                        goto out;
        }
 
        WARN_ON(recv->r_frag); /* leak! */
-       recv->r_frag = rds_ib_refill_one_frag(ic);
+       recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);
        if (!recv->r_frag)
                goto out;
 
@@ -344,7 +353,7 @@ out:
  *
  * -1 is returned if posting fails due to temporary resource exhaustion.
  */
-int rds_ib_recv_refill(struct rds_connection *conn, int prefill)
+void rds_ib_recv_refill(struct rds_connection *conn, int prefill)
 {
        struct rds_ib_connection *ic = conn->c_transport_data;
        struct rds_ib_recv_work *recv;
@@ -358,14 +367,12 @@ int rds_ib_recv_refill(struct rds_connection *conn, int prefill)
                if (pos >= ic->i_recv_ring.w_nr) {
                        printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
                                        pos);
-                       ret = -EINVAL;
                        break;
                }
 
                recv = &ic->i_recvs[pos];
-               ret = rds_ib_recv_refill_one(conn, recv);
+               ret = rds_ib_recv_refill_one(conn, recv, prefill);
                if (ret) {
-                       ret = -1;
                        break;
                }
 
@@ -379,7 +386,6 @@ int rds_ib_recv_refill(struct rds_connection *conn, int prefill)
                               "%pI4 returned %d, disconnecting and "
                               "reconnecting\n", &conn->c_faddr,
                               ret);
-                       ret = -1;
                        break;
                }
 
@@ -392,7 +398,6 @@ int rds_ib_recv_refill(struct rds_connection *conn, int prefill)
 
        if (ret)
                rds_ib_ring_unalloc(&ic->i_recv_ring, 1);
-       return ret;
 }
 
 /*
@@ -974,18 +979,22 @@ static inline void rds_poll_cq(struct rds_ib_connection *ic,
                 * to get a recv completion _before_ the rdmacm ESTABLISHED
                 * event is processed.
                 */
-               if (rds_conn_up(conn) || rds_conn_connecting(conn)) {
+               if (wc.status == IB_WC_SUCCESS) {
+                       rds_ib_process_recv(conn, recv, wc.byte_len, state);
+               } else {
                        /* We expect errors as the qp is drained during shutdown */
-                       if (wc.status == IB_WC_SUCCESS) {
-                               rds_ib_process_recv(conn, recv, wc.byte_len, state);
-                       } else {
+                       if (rds_conn_up(conn) || rds_conn_connecting(conn))
                                rds_ib_conn_error(conn, "recv completion on "
-                                      "%pI4 had status %u, disconnecting and "
-                                      "reconnecting\n", &conn->c_faddr,
-                                      wc.status);
-                       }
+                                                 "%pI4 had status %u, disconnecting and "
+                                                 "reconnecting\n", &conn->c_faddr,
+                                                 wc.status);
                }
 
+               /*
+                * It's very important that we only free this ring entry if we've truly
+                * freed the resources allocated to the entry.  The refilling path can
+                * leak if we don't.
+                */
                rds_ib_ring_free(&ic->i_recv_ring, 1);
        }
 }
@@ -1042,13 +1051,13 @@ int __init rds_ib_recv_init(void)
 
        rds_ib_incoming_slab = kmem_cache_create("rds_ib_incoming",
                                        sizeof(struct rds_ib_incoming),
-                                       0, 0, NULL);
+                                       0, SLAB_HWCACHE_ALIGN, NULL);
        if (!rds_ib_incoming_slab)
                goto out;
 
        rds_ib_frag_slab = kmem_cache_create("rds_ib_frag",
                                        sizeof(struct rds_page_frag),
-                                       0, 0, NULL);
+                                       0, SLAB_HWCACHE_ALIGN, NULL);
        if (!rds_ib_frag_slab)
                kmem_cache_destroy(rds_ib_incoming_slab);
        else