#include <asm/page.h>
#include <linux/mlx4/cq.h>
+#include <linux/slab.h>
#include <linux/mlx4/qp.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
+#include <linux/tcp.h>
#include "mlx4_en.h"
enum {
MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
+ MAX_BF = 256,
};
static int inline_thold __read_mostly = MAX_INLINE;
module_param_named(inline_thold, inline_thold, int, 0444);
-MODULE_PARM_DESC(inline_thold, "treshold for using inline data");
+MODULE_PARM_DESC(inline_thold, "threshold for using inline data");
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
- struct mlx4_en_tx_ring *ring, u32 size,
+ struct mlx4_en_tx_ring *ring, int qpn, u32 size,
u16 stride)
{
struct mlx4_en_dev *mdev = priv->mdev;
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
- err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn);
- if (err) {
- en_err(priv, "Failed reserving qp for tx ring.\n");
- goto err_map;
- }
-
+ ring->qpn = qpn;
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
- goto err_reserve;
+ goto err_map;
}
ring->qp.event = mlx4_en_sqp_event;
+ err = mlx4_bf_alloc(mdev->dev, &ring->bf);
+ if (err) {
+ en_dbg(DRV, priv, "working without blueflame (%d)", err);
+ ring->bf.uar = &mdev->priv_uar;
+ ring->bf.uar->map = mdev->uar_map;
+ ring->bf_enabled = false;
+ } else
+ ring->bf_enabled = true;
+
return 0;
-err_reserve:
- mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
struct mlx4_en_dev *mdev = priv->mdev;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
+ if (ring->bf_enabled)
+ mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
- int cq, int srqn)
+ int cq)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
memset(ring->buf, 0, ring->buf_size);
ring->qp_state = MLX4_QP_STATE_RST;
- ring->doorbell_qpn = swab32(ring->qp.qpn << 8);
+ ring->doorbell_qpn = ring->qp.qpn << 8;
mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn,
- ring->cqn, srqn, &ring->context);
+ ring->cqn, &ring->context);
+ if (ring->bf_enabled)
+ ring->context.usr_page = cpu_to_be32(ring->bf.uar->index);
err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context,
&ring->qp, &ring->qp_state);
} else {
if (!tx_info->inl) {
if ((void *) data >= end) {
- data = (struct mlx4_wqe_data_seg *)
- (ring->buf + ((void *) data - end));
+ data = ring->buf + ((void *)data - end);
}
if (tx_info->linear) {
for (i = 0; i < frags; i++) {
/* Check for wraparound before unmapping */
if ((void *) data >= end)
- data = (struct mlx4_wqe_data_seg *) ring->buf;
+ data = ring->buf;
frag = &skb_shinfo(skb)->frags[i];
pci_unmap_page(mdev->pdev,
(dma_addr_t) be64_to_cpu(data->addr),
frag->size, PCI_DMA_TODEVICE);
+ ++data;
}
}
/* Stamp the freed descriptor */
INC_PERF_COUNTER(priv->pstats.tx_poll);
- if (!spin_trylock(&ring->comp_lock)) {
+ if (!spin_trylock_irq(&ring->comp_lock)) {
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
return;
}
if (inflight && priv->port_up)
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
- spin_unlock(&ring->comp_lock);
+ spin_unlock_irq(&ring->comp_lock);
}
static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
{
struct mlx4_en_cq *cq = &priv->tx_cq[tx_ind];
struct mlx4_en_tx_ring *ring = &priv->tx_ring[tx_ind];
+ unsigned long flags;
/* If we don't have a pending timer, set one up to catch our recent
post in case the interface becomes idle */
/* Poll the CQ every mlx4_en_TX_MODER_POLL packets */
if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0)
- if (spin_trylock(&ring->comp_lock)) {
+ if (spin_trylock_irqsave(&ring->comp_lock, flags)) {
mlx4_en_process_tx_cq(priv->dev, cq);
- spin_unlock(&ring->comp_lock);
+ spin_unlock_irqrestore(&ring->comp_lock, flags);
}
}
else {
if (netif_msg_tx_err(priv))
en_warn(priv, "Non-linear headers\n");
- dev_kfree_skb_any(skb);
return 0;
}
}
- if (unlikely(*lso_header_size > MAX_LSO_HDR_SIZE)) {
- if (netif_msg_tx_err(priv))
- en_warn(priv, "LSO header size too big\n");
- dev_kfree_skb_any(skb);
- return 0;
- }
} else {
*lso_header_size = 0;
if (!is_inline(skb, NULL))
/* If we support per priority flow control and the packet contains
* a vlan tag, send the packet to the TX ring assigned to that priority
*/
- if (priv->prof->rx_ppp && priv->vlgrp && vlan_tx_tag_present(skb)) {
+ if (priv->prof->rx_ppp && vlan_tx_tag_present(skb)) {
vlan_tag = vlan_tx_tag_get(skb);
return MLX4_EN_NUM_TX_RINGS + (vlan_tag >> 13);
}
return skb_tx_hash(dev, skb);
}
-int mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
+static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, unsigned bytecnt)
+{
+ __iowrite64_copy(dst, src, bytecnt / 8);
+}
+
+netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_wqe_data_seg *data;
struct skb_frag_struct *frag;
struct mlx4_en_tx_info *tx_info;
+ struct ethhdr *ethh;
+ u64 mac;
+ u32 mac_l, mac_h;
int tx_ind = 0;
int nr_txbb;
int desc_size;
int real_size;
dma_addr_t dma;
- u32 index;
+ u32 index, bf_index;
__be32 op_own;
u16 vlan_tag = 0;
int i;
int lso_header_size;
void *fragptr;
+ bool bounce = false;
+
+ if (!priv->port_up)
+ goto tx_drop;
- if (unlikely(!skb->len)) {
- dev_kfree_skb_any(skb);
- return NETDEV_TX_OK;
- }
real_size = get_real_size(skb, dev, &lso_header_size);
if (unlikely(!real_size))
- return NETDEV_TX_OK;
+ goto tx_drop;
- /* Allign descriptor to TXBB size */
+ /* Align descriptor to TXBB size */
desc_size = ALIGN(real_size, TXBB_SIZE);
nr_txbb = desc_size / TXBB_SIZE;
if (unlikely(nr_txbb > MAX_DESC_TXBBS)) {
if (netif_msg_tx_err(priv))
en_warn(priv, "Oversized header or SG list\n");
- dev_kfree_skb_any(skb);
- return NETDEV_TX_OK;
+ goto tx_drop;
}
tx_ind = skb->queue_mapping;
ring = &priv->tx_ring[tx_ind];
- if (priv->vlgrp && vlan_tx_tag_present(skb))
+ if (vlan_tx_tag_present(skb))
vlan_tag = vlan_tx_tag_get(skb);
/* Check available TXBBs And 2K spare for prefetch */
return NETDEV_TX_BUSY;
}
- /* Now that we know what Tx ring to use */
- if (unlikely(!priv->port_up)) {
- if (netif_msg_tx_err(priv))
- en_warn(priv, "xmit: port down!\n");
- dev_kfree_skb_any(skb);
- return NETDEV_TX_OK;
- }
-
/* Track current inflight packets for performance analysis */
AVG_PERF_COUNTER(priv->pstats.inflight_avg,
(u32) (ring->prod - ring->cons - 1));
/* Packet is good - grab an index and transmit it */
index = ring->prod & ring->size_mask;
+ bf_index = ring->prod;
/* See if we have enough space for whole descriptor TXBB for setting
* SW ownership on next descriptor; if not, use a bounce buffer. */
if (likely(index + nr_txbb <= ring->size))
tx_desc = ring->buf + index * TXBB_SIZE;
- else
+ else {
tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf;
+ bounce = true;
+ }
/* Save skb in tx_info ring */
tx_info = &ring->tx_info[index];
priv->port_stats.tx_chksum_offload++;
}
+ if (unlikely(priv->validate_loopback)) {
+ /* Copy dst mac address to wqe */
+ skb_reset_mac_header(skb);
+ ethh = eth_hdr(skb);
+ if (ethh && ethh->h_dest) {
+ mac = mlx4_en_mac_to_u64(ethh->h_dest);
+ mac_h = (u32) ((mac & 0xffff00000000ULL) >> 16);
+ mac_l = (u32) (mac & 0xffffffff);
+ tx_desc->ctrl.srcrb_flags |= cpu_to_be32(mac_h);
+ tx_desc->ctrl.imm = cpu_to_be32(mac_l);
+ }
+ }
+
/* Handle LSO (TSO) packets */
if (lso_header_size) {
/* Mark opcode as LSO */
ring->prod += nr_txbb;
/* If we used a bounce buffer then copy descriptor back into place */
- if (tx_desc == (struct mlx4_en_tx_desc *) ring->bounce_buf)
+ if (bounce)
tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size);
/* Run destructor before passing skb to HW */
if (likely(!skb_shared(skb)))
skb_orphan(skb);
- /* Ensure new descirptor hits memory
- * before setting ownership of this descriptor to HW */
- wmb();
- tx_desc->ctrl.owner_opcode = op_own;
+ if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && !vlan_tag) {
+ *(__be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn);
+ op_own |= htonl((bf_index & 0xffff) << 8);
+ /* Ensure new descirptor hits memory
+ * before setting ownership of this descriptor to HW */
+ wmb();
+ tx_desc->ctrl.owner_opcode = op_own;
- /* Ring doorbell! */
- wmb();
- writel(ring->doorbell_qpn, mdev->uar_map + MLX4_SEND_DOORBELL);
+ wmb();
+
+ mlx4_bf_copy(ring->bf.reg + ring->bf.offset, (unsigned long *) &tx_desc->ctrl,
+ desc_size);
+
+ wmb();
+
+ ring->bf.offset ^= ring->bf.buf_size;
+ } else {
+ /* Ensure new descirptor hits memory
+ * before setting ownership of this descriptor to HW */
+ wmb();
+ tx_desc->ctrl.owner_opcode = op_own;
+ wmb();
+ iowrite32be(ring->doorbell_qpn, ring->bf.uar->map + MLX4_SEND_DOORBELL);
+ }
/* Poll CQ here */
mlx4_en_xmit_poll(priv, tx_ind);
- return 0;
+ return NETDEV_TX_OK;
+
+tx_drop:
+ dev_kfree_skb_any(skb);
+ priv->stats.tx_dropped++;
+ return NETDEV_TX_OK;
}
Code Review / linux-2.6.git / blobdiff