mlx4: Traffic steering management support for SRIOV
[linux-3.10.git] / drivers / net / ethernet / mellanox / mlx4 / main.c
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/errno.h>
39 #include <linux/pci.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/slab.h>
42 #include <linux/io-mapping.h>
43
44 #include <linux/mlx4/device.h>
45 #include <linux/mlx4/doorbell.h>
46
47 #include "mlx4.h"
48 #include "fw.h"
49 #include "icm.h"
50
51 MODULE_AUTHOR("Roland Dreier");
52 MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
53 MODULE_LICENSE("Dual BSD/GPL");
54 MODULE_VERSION(DRV_VERSION);
55
56 struct workqueue_struct *mlx4_wq;
57
58 #ifdef CONFIG_MLX4_DEBUG
59
60 int mlx4_debug_level = 0;
61 module_param_named(debug_level, mlx4_debug_level, int, 0644);
62 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
63
64 #endif /* CONFIG_MLX4_DEBUG */
65
66 #ifdef CONFIG_PCI_MSI
67
68 static int msi_x = 1;
69 module_param(msi_x, int, 0444);
70 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
71
72 #else /* CONFIG_PCI_MSI */
73
74 #define msi_x (0)
75
76 #endif /* CONFIG_PCI_MSI */
77
78 int mlx4_log_num_mgm_entry_size = 10;
79 module_param_named(log_num_mgm_entry_size,
80                         mlx4_log_num_mgm_entry_size, int, 0444);
81 MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num"
82                                          " of qp per mcg, for example:"
83                                          " 10 gives 248.range: 9<="
84                                          " log_num_mgm_entry_size <= 12");
85
86 static char mlx4_version[] __devinitdata =
87         DRV_NAME ": Mellanox ConnectX core driver v"
88         DRV_VERSION " (" DRV_RELDATE ")\n";
89
90 static struct mlx4_profile default_profile = {
91         .num_qp         = 1 << 17,
92         .num_srq        = 1 << 16,
93         .rdmarc_per_qp  = 1 << 4,
94         .num_cq         = 1 << 16,
95         .num_mcg        = 1 << 13,
96         .num_mpt        = 1 << 17,
97         .num_mtt        = 1 << 20,
98 };
99
100 static int log_num_mac = 2;
101 module_param_named(log_num_mac, log_num_mac, int, 0444);
102 MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)");
103
104 static int log_num_vlan;
105 module_param_named(log_num_vlan, log_num_vlan, int, 0444);
106 MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)");
107 /* Log2 max number of VLANs per ETH port (0-7) */
108 #define MLX4_LOG_NUM_VLANS 7
109
110 static int use_prio;
111 module_param_named(use_prio, use_prio, bool, 0444);
112 MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
113                   "(0/1, default 0)");
114
115 static int log_mtts_per_seg = ilog2(MLX4_MTT_ENTRY_PER_SEG);
116 module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444);
117 MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-7)");
118
119 int mlx4_check_port_params(struct mlx4_dev *dev,
120                            enum mlx4_port_type *port_type)
121 {
122         int i;
123
124         for (i = 0; i < dev->caps.num_ports - 1; i++) {
125                 if (port_type[i] != port_type[i + 1]) {
126                         if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
127                                 mlx4_err(dev, "Only same port types supported "
128                                          "on this HCA, aborting.\n");
129                                 return -EINVAL;
130                         }
131                         if (port_type[i] == MLX4_PORT_TYPE_ETH &&
132                             port_type[i + 1] == MLX4_PORT_TYPE_IB)
133                                 return -EINVAL;
134                 }
135         }
136
137         for (i = 0; i < dev->caps.num_ports; i++) {
138                 if (!(port_type[i] & dev->caps.supported_type[i+1])) {
139                         mlx4_err(dev, "Requested port type for port %d is not "
140                                       "supported on this HCA\n", i + 1);
141                         return -EINVAL;
142                 }
143         }
144         return 0;
145 }
146
147 static void mlx4_set_port_mask(struct mlx4_dev *dev)
148 {
149         int i;
150
151         for (i = 1; i <= dev->caps.num_ports; ++i)
152                 dev->caps.port_mask[i] = dev->caps.port_type[i];
153 }
154
155 static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
156 {
157         int err;
158         int i;
159
160         err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
161         if (err) {
162                 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
163                 return err;
164         }
165
166         if (dev_cap->min_page_sz > PAGE_SIZE) {
167                 mlx4_err(dev, "HCA minimum page size of %d bigger than "
168                          "kernel PAGE_SIZE of %ld, aborting.\n",
169                          dev_cap->min_page_sz, PAGE_SIZE);
170                 return -ENODEV;
171         }
172         if (dev_cap->num_ports > MLX4_MAX_PORTS) {
173                 mlx4_err(dev, "HCA has %d ports, but we only support %d, "
174                          "aborting.\n",
175                          dev_cap->num_ports, MLX4_MAX_PORTS);
176                 return -ENODEV;
177         }
178
179         if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) {
180                 mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than "
181                          "PCI resource 2 size of 0x%llx, aborting.\n",
182                          dev_cap->uar_size,
183                          (unsigned long long) pci_resource_len(dev->pdev, 2));
184                 return -ENODEV;
185         }
186
187         dev->caps.num_ports          = dev_cap->num_ports;
188         for (i = 1; i <= dev->caps.num_ports; ++i) {
189                 dev->caps.vl_cap[i]         = dev_cap->max_vl[i];
190                 dev->caps.ib_mtu_cap[i]     = dev_cap->ib_mtu[i];
191                 dev->caps.gid_table_len[i]  = dev_cap->max_gids[i];
192                 dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i];
193                 dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
194                 dev->caps.eth_mtu_cap[i]    = dev_cap->eth_mtu[i];
195                 dev->caps.def_mac[i]        = dev_cap->def_mac[i];
196                 dev->caps.supported_type[i] = dev_cap->supported_port_types[i];
197                 dev->caps.trans_type[i]     = dev_cap->trans_type[i];
198                 dev->caps.vendor_oui[i]     = dev_cap->vendor_oui[i];
199                 dev->caps.wavelength[i]     = dev_cap->wavelength[i];
200                 dev->caps.trans_code[i]     = dev_cap->trans_code[i];
201         }
202
203         dev->caps.num_uars           = dev_cap->uar_size / PAGE_SIZE;
204         dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
205         dev->caps.bf_reg_size        = dev_cap->bf_reg_size;
206         dev->caps.bf_regs_per_page   = dev_cap->bf_regs_per_page;
207         dev->caps.max_sq_sg          = dev_cap->max_sq_sg;
208         dev->caps.max_rq_sg          = dev_cap->max_rq_sg;
209         dev->caps.max_wqes           = dev_cap->max_qp_sz;
210         dev->caps.max_qp_init_rdma   = dev_cap->max_requester_per_qp;
211         dev->caps.max_srq_wqes       = dev_cap->max_srq_sz;
212         dev->caps.max_srq_sge        = dev_cap->max_rq_sg - 1;
213         dev->caps.reserved_srqs      = dev_cap->reserved_srqs;
214         dev->caps.max_sq_desc_sz     = dev_cap->max_sq_desc_sz;
215         dev->caps.max_rq_desc_sz     = dev_cap->max_rq_desc_sz;
216         dev->caps.num_qp_per_mgm     = mlx4_get_qp_per_mgm(dev);
217         /*
218          * Subtract 1 from the limit because we need to allocate a
219          * spare CQE so the HCA HW can tell the difference between an
220          * empty CQ and a full CQ.
221          */
222         dev->caps.max_cqes           = dev_cap->max_cq_sz - 1;
223         dev->caps.reserved_cqs       = dev_cap->reserved_cqs;
224         dev->caps.reserved_eqs       = dev_cap->reserved_eqs;
225         dev->caps.mtts_per_seg       = 1 << log_mtts_per_seg;
226         dev->caps.reserved_mtts      = DIV_ROUND_UP(dev_cap->reserved_mtts,
227                                                     dev->caps.mtts_per_seg);
228         dev->caps.reserved_mrws      = dev_cap->reserved_mrws;
229         dev->caps.reserved_uars      = dev_cap->reserved_uars;
230         dev->caps.reserved_pds       = dev_cap->reserved_pds;
231         dev->caps.reserved_xrcds     = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
232                                         dev_cap->reserved_xrcds : 0;
233         dev->caps.max_xrcds          = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
234                                         dev_cap->max_xrcds : 0;
235         dev->caps.mtt_entry_sz       = dev->caps.mtts_per_seg * dev_cap->mtt_entry_sz;
236         dev->caps.max_msg_sz         = dev_cap->max_msg_sz;
237         dev->caps.page_size_cap      = ~(u32) (dev_cap->min_page_sz - 1);
238         dev->caps.flags              = dev_cap->flags;
239         dev->caps.bmme_flags         = dev_cap->bmme_flags;
240         dev->caps.reserved_lkey      = dev_cap->reserved_lkey;
241         dev->caps.stat_rate_support  = dev_cap->stat_rate_support;
242         dev->caps.max_gso_sz         = dev_cap->max_gso_sz;
243
244         dev->caps.log_num_macs  = log_num_mac;
245         dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS;
246         dev->caps.log_num_prios = use_prio ? 3 : 0;
247
248         for (i = 1; i <= dev->caps.num_ports; ++i) {
249                 if (dev->caps.supported_type[i] != MLX4_PORT_TYPE_ETH)
250                         dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
251                 else
252                         dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
253                 dev->caps.possible_type[i] = dev->caps.port_type[i];
254                 mlx4_priv(dev)->sense.sense_allowed[i] =
255                         dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO;
256
257                 if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
258                         dev->caps.log_num_macs = dev_cap->log_max_macs[i];
259                         mlx4_warn(dev, "Requested number of MACs is too much "
260                                   "for port %d, reducing to %d.\n",
261                                   i, 1 << dev->caps.log_num_macs);
262                 }
263                 if (dev->caps.log_num_vlans > dev_cap->log_max_vlans[i]) {
264                         dev->caps.log_num_vlans = dev_cap->log_max_vlans[i];
265                         mlx4_warn(dev, "Requested number of VLANs is too much "
266                                   "for port %d, reducing to %d.\n",
267                                   i, 1 << dev->caps.log_num_vlans);
268                 }
269         }
270
271         mlx4_set_port_mask(dev);
272
273         dev->caps.max_counters = 1 << ilog2(dev_cap->max_counters);
274
275         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
276         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
277                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
278                 (1 << dev->caps.log_num_macs) *
279                 (1 << dev->caps.log_num_vlans) *
280                 (1 << dev->caps.log_num_prios) *
281                 dev->caps.num_ports;
282         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
283
284         dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
285                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
286                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +
287                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
288
289         return 0;
290 }
291
292 /*
293  * Change the port configuration of the device.
294  * Every user of this function must hold the port mutex.
295  */
296 int mlx4_change_port_types(struct mlx4_dev *dev,
297                            enum mlx4_port_type *port_types)
298 {
299         int err = 0;
300         int change = 0;
301         int port;
302
303         for (port = 0; port <  dev->caps.num_ports; port++) {
304                 /* Change the port type only if the new type is different
305                  * from the current, and not set to Auto */
306                 if (port_types[port] != dev->caps.port_type[port + 1]) {
307                         change = 1;
308                         dev->caps.port_type[port + 1] = port_types[port];
309                 }
310         }
311         if (change) {
312                 mlx4_unregister_device(dev);
313                 for (port = 1; port <= dev->caps.num_ports; port++) {
314                         mlx4_CLOSE_PORT(dev, port);
315                         err = mlx4_SET_PORT(dev, port);
316                         if (err) {
317                                 mlx4_err(dev, "Failed to set port %d, "
318                                               "aborting\n", port);
319                                 goto out;
320                         }
321                 }
322                 mlx4_set_port_mask(dev);
323                 err = mlx4_register_device(dev);
324         }
325
326 out:
327         return err;
328 }
329
330 static ssize_t show_port_type(struct device *dev,
331                               struct device_attribute *attr,
332                               char *buf)
333 {
334         struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
335                                                    port_attr);
336         struct mlx4_dev *mdev = info->dev;
337         char type[8];
338
339         sprintf(type, "%s",
340                 (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
341                 "ib" : "eth");
342         if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
343                 sprintf(buf, "auto (%s)\n", type);
344         else
345                 sprintf(buf, "%s\n", type);
346
347         return strlen(buf);
348 }
349
350 static ssize_t set_port_type(struct device *dev,
351                              struct device_attribute *attr,
352                              const char *buf, size_t count)
353 {
354         struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
355                                                    port_attr);
356         struct mlx4_dev *mdev = info->dev;
357         struct mlx4_priv *priv = mlx4_priv(mdev);
358         enum mlx4_port_type types[MLX4_MAX_PORTS];
359         enum mlx4_port_type new_types[MLX4_MAX_PORTS];
360         int i;
361         int err = 0;
362
363         if (!strcmp(buf, "ib\n"))
364                 info->tmp_type = MLX4_PORT_TYPE_IB;
365         else if (!strcmp(buf, "eth\n"))
366                 info->tmp_type = MLX4_PORT_TYPE_ETH;
367         else if (!strcmp(buf, "auto\n"))
368                 info->tmp_type = MLX4_PORT_TYPE_AUTO;
369         else {
370                 mlx4_err(mdev, "%s is not supported port type\n", buf);
371                 return -EINVAL;
372         }
373
374         mlx4_stop_sense(mdev);
375         mutex_lock(&priv->port_mutex);
376         /* Possible type is always the one that was delivered */
377         mdev->caps.possible_type[info->port] = info->tmp_type;
378
379         for (i = 0; i < mdev->caps.num_ports; i++) {
380                 types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
381                                         mdev->caps.possible_type[i+1];
382                 if (types[i] == MLX4_PORT_TYPE_AUTO)
383                         types[i] = mdev->caps.port_type[i+1];
384         }
385
386         if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
387                 for (i = 1; i <= mdev->caps.num_ports; i++) {
388                         if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
389                                 mdev->caps.possible_type[i] = mdev->caps.port_type[i];
390                                 err = -EINVAL;
391                         }
392                 }
393         }
394         if (err) {
395                 mlx4_err(mdev, "Auto sensing is not supported on this HCA. "
396                                "Set only 'eth' or 'ib' for both ports "
397                                "(should be the same)\n");
398                 goto out;
399         }
400
401         mlx4_do_sense_ports(mdev, new_types, types);
402
403         err = mlx4_check_port_params(mdev, new_types);
404         if (err)
405                 goto out;
406
407         /* We are about to apply the changes after the configuration
408          * was verified, no need to remember the temporary types
409          * any more */
410         for (i = 0; i < mdev->caps.num_ports; i++)
411                 priv->port[i + 1].tmp_type = 0;
412
413         err = mlx4_change_port_types(mdev, new_types);
414
415 out:
416         mlx4_start_sense(mdev);
417         mutex_unlock(&priv->port_mutex);
418         return err ? err : count;
419 }
420
421 static int mlx4_load_fw(struct mlx4_dev *dev)
422 {
423         struct mlx4_priv *priv = mlx4_priv(dev);
424         int err;
425
426         priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,
427                                          GFP_HIGHUSER | __GFP_NOWARN, 0);
428         if (!priv->fw.fw_icm) {
429                 mlx4_err(dev, "Couldn't allocate FW area, aborting.\n");
430                 return -ENOMEM;
431         }
432
433         err = mlx4_MAP_FA(dev, priv->fw.fw_icm);
434         if (err) {
435                 mlx4_err(dev, "MAP_FA command failed, aborting.\n");
436                 goto err_free;
437         }
438
439         err = mlx4_RUN_FW(dev);
440         if (err) {
441                 mlx4_err(dev, "RUN_FW command failed, aborting.\n");
442                 goto err_unmap_fa;
443         }
444
445         return 0;
446
447 err_unmap_fa:
448         mlx4_UNMAP_FA(dev);
449
450 err_free:
451         mlx4_free_icm(dev, priv->fw.fw_icm, 0);
452         return err;
453 }
454
455 static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
456                                 int cmpt_entry_sz)
457 {
458         struct mlx4_priv *priv = mlx4_priv(dev);
459         int err;
460
461         err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table,
462                                   cmpt_base +
463                                   ((u64) (MLX4_CMPT_TYPE_QP *
464                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
465                                   cmpt_entry_sz, dev->caps.num_qps,
466                                   dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
467                                   0, 0);
468         if (err)
469                 goto err;
470
471         err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table,
472                                   cmpt_base +
473                                   ((u64) (MLX4_CMPT_TYPE_SRQ *
474                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
475                                   cmpt_entry_sz, dev->caps.num_srqs,
476                                   dev->caps.reserved_srqs, 0, 0);
477         if (err)
478                 goto err_qp;
479
480         err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table,
481                                   cmpt_base +
482                                   ((u64) (MLX4_CMPT_TYPE_CQ *
483                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
484                                   cmpt_entry_sz, dev->caps.num_cqs,
485                                   dev->caps.reserved_cqs, 0, 0);
486         if (err)
487                 goto err_srq;
488
489         err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table,
490                                   cmpt_base +
491                                   ((u64) (MLX4_CMPT_TYPE_EQ *
492                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
493                                   cmpt_entry_sz,
494                                   dev->caps.num_eqs, dev->caps.num_eqs, 0, 0);
495         if (err)
496                 goto err_cq;
497
498         return 0;
499
500 err_cq:
501         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
502
503 err_srq:
504         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
505
506 err_qp:
507         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
508
509 err:
510         return err;
511 }
512
513 static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
514                          struct mlx4_init_hca_param *init_hca, u64 icm_size)
515 {
516         struct mlx4_priv *priv = mlx4_priv(dev);
517         u64 aux_pages;
518         int err;
519
520         err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
521         if (err) {
522                 mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n");
523                 return err;
524         }
525
526         mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n",
527                  (unsigned long long) icm_size >> 10,
528                  (unsigned long long) aux_pages << 2);
529
530         priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
531                                           GFP_HIGHUSER | __GFP_NOWARN, 0);
532         if (!priv->fw.aux_icm) {
533                 mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n");
534                 return -ENOMEM;
535         }
536
537         err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
538         if (err) {
539                 mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n");
540                 goto err_free_aux;
541         }
542
543         err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
544         if (err) {
545                 mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n");
546                 goto err_unmap_aux;
547         }
548
549         err = mlx4_init_icm_table(dev, &priv->eq_table.table,
550                                   init_hca->eqc_base, dev_cap->eqc_entry_sz,
551                                   dev->caps.num_eqs, dev->caps.num_eqs,
552                                   0, 0);
553         if (err) {
554                 mlx4_err(dev, "Failed to map EQ context memory, aborting.\n");
555                 goto err_unmap_cmpt;
556         }
557
558         /*
559          * Reserved MTT entries must be aligned up to a cacheline
560          * boundary, since the FW will write to them, while the driver
561          * writes to all other MTT entries. (The variable
562          * dev->caps.mtt_entry_sz below is really the MTT segment
563          * size, not the raw entry size)
564          */
565         dev->caps.reserved_mtts =
566                 ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
567                       dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
568
569         err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
570                                   init_hca->mtt_base,
571                                   dev->caps.mtt_entry_sz,
572                                   dev->caps.num_mtt_segs,
573                                   dev->caps.reserved_mtts, 1, 0);
574         if (err) {
575                 mlx4_err(dev, "Failed to map MTT context memory, aborting.\n");
576                 goto err_unmap_eq;
577         }
578
579         err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
580                                   init_hca->dmpt_base,
581                                   dev_cap->dmpt_entry_sz,
582                                   dev->caps.num_mpts,
583                                   dev->caps.reserved_mrws, 1, 1);
584         if (err) {
585                 mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n");
586                 goto err_unmap_mtt;
587         }
588
589         err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
590                                   init_hca->qpc_base,
591                                   dev_cap->qpc_entry_sz,
592                                   dev->caps.num_qps,
593                                   dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
594                                   0, 0);
595         if (err) {
596                 mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
597                 goto err_unmap_dmpt;
598         }
599
600         err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
601                                   init_hca->auxc_base,
602                                   dev_cap->aux_entry_sz,
603                                   dev->caps.num_qps,
604                                   dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
605                                   0, 0);
606         if (err) {
607                 mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
608                 goto err_unmap_qp;
609         }
610
611         err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
612                                   init_hca->altc_base,
613                                   dev_cap->altc_entry_sz,
614                                   dev->caps.num_qps,
615                                   dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
616                                   0, 0);
617         if (err) {
618                 mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
619                 goto err_unmap_auxc;
620         }
621
622         err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
623                                   init_hca->rdmarc_base,
624                                   dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
625                                   dev->caps.num_qps,
626                                   dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
627                                   0, 0);
628         if (err) {
629                 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
630                 goto err_unmap_altc;
631         }
632
633         err = mlx4_init_icm_table(dev, &priv->cq_table.table,
634                                   init_hca->cqc_base,
635                                   dev_cap->cqc_entry_sz,
636                                   dev->caps.num_cqs,
637                                   dev->caps.reserved_cqs, 0, 0);
638         if (err) {
639                 mlx4_err(dev, "Failed to map CQ context memory, aborting.\n");
640                 goto err_unmap_rdmarc;
641         }
642
643         err = mlx4_init_icm_table(dev, &priv->srq_table.table,
644                                   init_hca->srqc_base,
645                                   dev_cap->srq_entry_sz,
646                                   dev->caps.num_srqs,
647                                   dev->caps.reserved_srqs, 0, 0);
648         if (err) {
649                 mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n");
650                 goto err_unmap_cq;
651         }
652
653         /*
654          * It's not strictly required, but for simplicity just map the
655          * whole multicast group table now.  The table isn't very big
656          * and it's a lot easier than trying to track ref counts.
657          */
658         err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
659                                   init_hca->mc_base,
660                                   mlx4_get_mgm_entry_size(dev),
661                                   dev->caps.num_mgms + dev->caps.num_amgms,
662                                   dev->caps.num_mgms + dev->caps.num_amgms,
663                                   0, 0);
664         if (err) {
665                 mlx4_err(dev, "Failed to map MCG context memory, aborting.\n");
666                 goto err_unmap_srq;
667         }
668
669         return 0;
670
671 err_unmap_srq:
672         mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
673
674 err_unmap_cq:
675         mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
676
677 err_unmap_rdmarc:
678         mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
679
680 err_unmap_altc:
681         mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
682
683 err_unmap_auxc:
684         mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
685
686 err_unmap_qp:
687         mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
688
689 err_unmap_dmpt:
690         mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
691
692 err_unmap_mtt:
693         mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
694
695 err_unmap_eq:
696         mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
697
698 err_unmap_cmpt:
699         mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
700         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
701         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
702         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
703
704 err_unmap_aux:
705         mlx4_UNMAP_ICM_AUX(dev);
706
707 err_free_aux:
708         mlx4_free_icm(dev, priv->fw.aux_icm, 0);
709
710         return err;
711 }
712
713 static void mlx4_free_icms(struct mlx4_dev *dev)
714 {
715         struct mlx4_priv *priv = mlx4_priv(dev);
716
717         mlx4_cleanup_icm_table(dev, &priv->mcg_table.table);
718         mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
719         mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
720         mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
721         mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
722         mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
723         mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
724         mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
725         mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
726         mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
727         mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
728         mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
729         mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
730         mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
731
732         mlx4_UNMAP_ICM_AUX(dev);
733         mlx4_free_icm(dev, priv->fw.aux_icm, 0);
734 }
735
736 static int map_bf_area(struct mlx4_dev *dev)
737 {
738         struct mlx4_priv *priv = mlx4_priv(dev);
739         resource_size_t bf_start;
740         resource_size_t bf_len;
741         int err = 0;
742
743         bf_start = pci_resource_start(dev->pdev, 2) + (dev->caps.num_uars << PAGE_SHIFT);
744         bf_len = pci_resource_len(dev->pdev, 2) - (dev->caps.num_uars << PAGE_SHIFT);
745         priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len);
746         if (!priv->bf_mapping)
747                 err = -ENOMEM;
748
749         return err;
750 }
751
752 static void unmap_bf_area(struct mlx4_dev *dev)
753 {
754         if (mlx4_priv(dev)->bf_mapping)
755                 io_mapping_free(mlx4_priv(dev)->bf_mapping);
756 }
757
758 static void mlx4_close_hca(struct mlx4_dev *dev)
759 {
760         unmap_bf_area(dev);
761         mlx4_CLOSE_HCA(dev, 0);
762         mlx4_free_icms(dev);
763         mlx4_UNMAP_FA(dev);
764         mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
765 }
766
767 static int mlx4_init_hca(struct mlx4_dev *dev)
768 {
769         struct mlx4_priv          *priv = mlx4_priv(dev);
770         struct mlx4_adapter        adapter;
771         struct mlx4_dev_cap        dev_cap;
772         struct mlx4_mod_stat_cfg   mlx4_cfg;
773         struct mlx4_profile        profile;
774         struct mlx4_init_hca_param init_hca;
775         u64 icm_size;
776         int err;
777
778         err = mlx4_QUERY_FW(dev);
779         if (err) {
780                 if (err == -EACCES)
781                         mlx4_info(dev, "non-primary physical function, skipping.\n");
782                 else
783                         mlx4_err(dev, "QUERY_FW command failed, aborting.\n");
784                 return err;
785         }
786
787         err = mlx4_load_fw(dev);
788         if (err) {
789                 mlx4_err(dev, "Failed to start FW, aborting.\n");
790                 return err;
791         }
792
793         mlx4_cfg.log_pg_sz_m = 1;
794         mlx4_cfg.log_pg_sz = 0;
795         err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
796         if (err)
797                 mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
798
799         err = mlx4_dev_cap(dev, &dev_cap);
800         if (err) {
801                 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
802                 goto err_stop_fw;
803         }
804
805         profile = default_profile;
806
807         icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca);
808         if ((long long) icm_size < 0) {
809                 err = icm_size;
810                 goto err_stop_fw;
811         }
812
813         if (map_bf_area(dev))
814                 mlx4_dbg(dev, "Failed to map blue flame area\n");
815
816         init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
817
818         err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
819         if (err)
820                 goto err_stop_fw;
821
822         err = mlx4_INIT_HCA(dev, &init_hca);
823         if (err) {
824                 mlx4_err(dev, "INIT_HCA command failed, aborting.\n");
825                 goto err_free_icm;
826         }
827
828         err = mlx4_QUERY_ADAPTER(dev, &adapter);
829         if (err) {
830                 mlx4_err(dev, "QUERY_ADAPTER command failed, aborting.\n");
831                 goto err_close;
832         }
833
834         priv->eq_table.inta_pin = adapter.inta_pin;
835         memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id);
836
837         return 0;
838
839 err_close:
840         mlx4_CLOSE_HCA(dev, 0);
841
842 err_free_icm:
843         mlx4_free_icms(dev);
844
845 err_stop_fw:
846         unmap_bf_area(dev);
847         mlx4_UNMAP_FA(dev);
848         mlx4_free_icm(dev, priv->fw.fw_icm, 0);
849
850         return err;
851 }
852
853 static int mlx4_init_counters_table(struct mlx4_dev *dev)
854 {
855         struct mlx4_priv *priv = mlx4_priv(dev);
856         int nent;
857
858         if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
859                 return -ENOENT;
860
861         nent = dev->caps.max_counters;
862         return mlx4_bitmap_init(&priv->counters_bitmap, nent, nent - 1, 0, 0);
863 }
864
865 static void mlx4_cleanup_counters_table(struct mlx4_dev *dev)
866 {
867         mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap);
868 }
869
870 int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
871 {
872         struct mlx4_priv *priv = mlx4_priv(dev);
873
874         if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
875                 return -ENOENT;
876
877         *idx = mlx4_bitmap_alloc(&priv->counters_bitmap);
878         if (*idx == -1)
879                 return -ENOMEM;
880
881         return 0;
882 }
883 EXPORT_SYMBOL_GPL(mlx4_counter_alloc);
884
885 void mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
886 {
887         mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx);
888         return;
889 }
890 EXPORT_SYMBOL_GPL(mlx4_counter_free);
891
892 static int mlx4_setup_hca(struct mlx4_dev *dev)
893 {
894         struct mlx4_priv *priv = mlx4_priv(dev);
895         int err;
896         int port;
897         __be32 ib_port_default_caps;
898
899         err = mlx4_init_uar_table(dev);
900         if (err) {
901                 mlx4_err(dev, "Failed to initialize "
902                          "user access region table, aborting.\n");
903                 return err;
904         }
905
906         err = mlx4_uar_alloc(dev, &priv->driver_uar);
907         if (err) {
908                 mlx4_err(dev, "Failed to allocate driver access region, "
909                          "aborting.\n");
910                 goto err_uar_table_free;
911         }
912
913         priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
914         if (!priv->kar) {
915                 mlx4_err(dev, "Couldn't map kernel access region, "
916                          "aborting.\n");
917                 err = -ENOMEM;
918                 goto err_uar_free;
919         }
920
921         err = mlx4_init_pd_table(dev);
922         if (err) {
923                 mlx4_err(dev, "Failed to initialize "
924                          "protection domain table, aborting.\n");
925                 goto err_kar_unmap;
926         }
927
928         err = mlx4_init_xrcd_table(dev);
929         if (err) {
930                 mlx4_err(dev, "Failed to initialize "
931                          "reliable connection domain table, aborting.\n");
932                 goto err_pd_table_free;
933         }
934
935         err = mlx4_init_mr_table(dev);
936         if (err) {
937                 mlx4_err(dev, "Failed to initialize "
938                          "memory region table, aborting.\n");
939                 goto err_xrcd_table_free;
940         }
941
942         err = mlx4_init_eq_table(dev);
943         if (err) {
944                 mlx4_err(dev, "Failed to initialize "
945                          "event queue table, aborting.\n");
946                 goto err_mr_table_free;
947         }
948
949         err = mlx4_cmd_use_events(dev);
950         if (err) {
951                 mlx4_err(dev, "Failed to switch to event-driven "
952                          "firmware commands, aborting.\n");
953                 goto err_eq_table_free;
954         }
955
956         err = mlx4_NOP(dev);
957         if (err) {
958                 if (dev->flags & MLX4_FLAG_MSI_X) {
959                         mlx4_warn(dev, "NOP command failed to generate MSI-X "
960                                   "interrupt IRQ %d).\n",
961                                   priv->eq_table.eq[dev->caps.num_comp_vectors].irq);
962                         mlx4_warn(dev, "Trying again without MSI-X.\n");
963                 } else {
964                         mlx4_err(dev, "NOP command failed to generate interrupt "
965                                  "(IRQ %d), aborting.\n",
966                                  priv->eq_table.eq[dev->caps.num_comp_vectors].irq);
967                         mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
968                 }
969
970                 goto err_cmd_poll;
971         }
972
973         mlx4_dbg(dev, "NOP command IRQ test passed\n");
974
975         err = mlx4_init_cq_table(dev);
976         if (err) {
977                 mlx4_err(dev, "Failed to initialize "
978                          "completion queue table, aborting.\n");
979                 goto err_cmd_poll;
980         }
981
982         err = mlx4_init_srq_table(dev);
983         if (err) {
984                 mlx4_err(dev, "Failed to initialize "
985                          "shared receive queue table, aborting.\n");
986                 goto err_cq_table_free;
987         }
988
989         err = mlx4_init_qp_table(dev);
990         if (err) {
991                 mlx4_err(dev, "Failed to initialize "
992                          "queue pair table, aborting.\n");
993                 goto err_srq_table_free;
994         }
995
996         err = mlx4_init_mcg_table(dev);
997         if (err) {
998                 mlx4_err(dev, "Failed to initialize "
999                          "multicast group table, aborting.\n");
1000                 goto err_qp_table_free;
1001         }
1002
1003         err = mlx4_init_counters_table(dev);
1004         if (err && err != -ENOENT) {
1005                 mlx4_err(dev, "Failed to initialize counters table, aborting.\n");
1006                 goto err_counters_table_free;
1007         }
1008
1009         for (port = 1; port <= dev->caps.num_ports; port++) {
1010                 enum mlx4_port_type port_type = 0;
1011                 mlx4_SENSE_PORT(dev, port, &port_type);
1012                 if (port_type)
1013                         dev->caps.port_type[port] = port_type;
1014                 ib_port_default_caps = 0;
1015                 err = mlx4_get_port_ib_caps(dev, port, &ib_port_default_caps);
1016                 if (err)
1017                         mlx4_warn(dev, "failed to get port %d default "
1018                                   "ib capabilities (%d). Continuing with "
1019                                   "caps = 0\n", port, err);
1020                 dev->caps.ib_port_def_cap[port] = ib_port_default_caps;
1021
1022                 err = mlx4_check_ext_port_caps(dev, port);
1023                 if (err)
1024                         mlx4_warn(dev, "failed to get port %d extended "
1025                                   "port capabilities support info (%d)."
1026                                   " Assuming not supported\n", port, err);
1027
1028                 err = mlx4_SET_PORT(dev, port);
1029                 if (err) {
1030                         mlx4_err(dev, "Failed to set port %d, aborting\n",
1031                                 port);
1032                         goto err_mcg_table_free;
1033                 }
1034         }
1035         mlx4_set_port_mask(dev);
1036
1037         return 0;
1038
1039 err_mcg_table_free:
1040         mlx4_cleanup_mcg_table(dev);
1041
1042 err_counters_table_free:
1043         mlx4_cleanup_counters_table(dev);
1044
1045 err_qp_table_free:
1046         mlx4_cleanup_qp_table(dev);
1047
1048 err_srq_table_free:
1049         mlx4_cleanup_srq_table(dev);
1050
1051 err_cq_table_free:
1052         mlx4_cleanup_cq_table(dev);
1053
1054 err_cmd_poll:
1055         mlx4_cmd_use_polling(dev);
1056
1057 err_eq_table_free:
1058         mlx4_cleanup_eq_table(dev);
1059
1060 err_mr_table_free:
1061         mlx4_cleanup_mr_table(dev);
1062
1063 err_xrcd_table_free:
1064         mlx4_cleanup_xrcd_table(dev);
1065
1066 err_pd_table_free:
1067         mlx4_cleanup_pd_table(dev);
1068
1069 err_kar_unmap:
1070         iounmap(priv->kar);
1071
1072 err_uar_free:
1073         mlx4_uar_free(dev, &priv->driver_uar);
1074
1075 err_uar_table_free:
1076         mlx4_cleanup_uar_table(dev);
1077         return err;
1078 }
1079
1080 static void mlx4_enable_msi_x(struct mlx4_dev *dev)
1081 {
1082         struct mlx4_priv *priv = mlx4_priv(dev);
1083         struct msix_entry *entries;
1084         int nreq = min_t(int, dev->caps.num_ports *
1085                          min_t(int, num_online_cpus() + 1, MAX_MSIX_P_PORT)
1086                                 + MSIX_LEGACY_SZ, MAX_MSIX);
1087         int err;
1088         int i;
1089
1090         if (msi_x) {
1091                 nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,
1092                              nreq);
1093                 entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL);
1094                 if (!entries)
1095                         goto no_msi;
1096
1097                 for (i = 0; i < nreq; ++i)
1098                         entries[i].entry = i;
1099
1100         retry:
1101                 err = pci_enable_msix(dev->pdev, entries, nreq);
1102                 if (err) {
1103                         /* Try again if at least 2 vectors are available */
1104                         if (err > 1) {
1105                                 mlx4_info(dev, "Requested %d vectors, "
1106                                           "but only %d MSI-X vectors available, "
1107                                           "trying again\n", nreq, err);
1108                                 nreq = err;
1109                                 goto retry;
1110                         }
1111                         kfree(entries);
1112                         goto no_msi;
1113                 }
1114
1115                 if (nreq <
1116                     MSIX_LEGACY_SZ + dev->caps.num_ports * MIN_MSIX_P_PORT) {
1117                         /*Working in legacy mode , all EQ's shared*/
1118                         dev->caps.comp_pool           = 0;
1119                         dev->caps.num_comp_vectors = nreq - 1;
1120                 } else {
1121                         dev->caps.comp_pool           = nreq - MSIX_LEGACY_SZ;
1122                         dev->caps.num_comp_vectors = MSIX_LEGACY_SZ - 1;
1123                 }
1124                 for (i = 0; i < nreq; ++i)
1125                         priv->eq_table.eq[i].irq = entries[i].vector;
1126
1127                 dev->flags |= MLX4_FLAG_MSI_X;
1128
1129                 kfree(entries);
1130                 return;
1131         }
1132
1133 no_msi:
1134         dev->caps.num_comp_vectors = 1;
1135         dev->caps.comp_pool        = 0;
1136
1137         for (i = 0; i < 2; ++i)
1138                 priv->eq_table.eq[i].irq = dev->pdev->irq;
1139 }
1140
1141 static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
1142 {
1143         struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
1144         int err = 0;
1145
1146         info->dev = dev;
1147         info->port = port;
1148         mlx4_init_mac_table(dev, &info->mac_table);
1149         mlx4_init_vlan_table(dev, &info->vlan_table);
1150         info->base_qpn = dev->caps.reserved_qps_base[MLX4_QP_REGION_ETH_ADDR] +
1151                         (port - 1) * (1 << log_num_mac);
1152
1153         sprintf(info->dev_name, "mlx4_port%d", port);
1154         info->port_attr.attr.name = info->dev_name;
1155         info->port_attr.attr.mode = S_IRUGO | S_IWUSR;
1156         info->port_attr.show      = show_port_type;
1157         info->port_attr.store     = set_port_type;
1158         sysfs_attr_init(&info->port_attr.attr);
1159
1160         err = device_create_file(&dev->pdev->dev, &info->port_attr);
1161         if (err) {
1162                 mlx4_err(dev, "Failed to create file for port %d\n", port);
1163                 info->port = -1;
1164         }
1165
1166         return err;
1167 }
1168
1169 static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
1170 {
1171         if (info->port < 0)
1172                 return;
1173
1174         device_remove_file(&info->dev->pdev->dev, &info->port_attr);
1175 }
1176
1177 static int mlx4_init_steering(struct mlx4_dev *dev)
1178 {
1179         struct mlx4_priv *priv = mlx4_priv(dev);
1180         int num_entries = dev->caps.num_ports;
1181         int i, j;
1182
1183         priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL);
1184         if (!priv->steer)
1185                 return -ENOMEM;
1186
1187         for (i = 0; i < num_entries; i++) {
1188                 for (j = 0; j < MLX4_NUM_STEERS; j++) {
1189                         INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]);
1190                         INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]);
1191                 }
1192                 INIT_LIST_HEAD(&priv->steer[i].high_prios);
1193         }
1194         return 0;
1195 }
1196
1197 static void mlx4_clear_steering(struct mlx4_dev *dev)
1198 {
1199         struct mlx4_priv *priv = mlx4_priv(dev);
1200         struct mlx4_steer_index *entry, *tmp_entry;
1201         struct mlx4_promisc_qp *pqp, *tmp_pqp;
1202         int num_entries = dev->caps.num_ports;
1203         int i, j;
1204
1205         for (i = 0; i < num_entries; i++) {
1206                 for (j = 0; j < MLX4_NUM_STEERS; j++) {
1207                         list_for_each_entry_safe(pqp, tmp_pqp,
1208                                                  &priv->steer[i].promisc_qps[j],
1209                                                  list) {
1210                                 list_del(&pqp->list);
1211                                 kfree(pqp);
1212                         }
1213                         list_for_each_entry_safe(entry, tmp_entry,
1214                                                  &priv->steer[i].steer_entries[j],
1215                                                  list) {
1216                                 list_del(&entry->list);
1217                                 list_for_each_entry_safe(pqp, tmp_pqp,
1218                                                          &entry->duplicates,
1219                                                          list) {
1220                                         list_del(&pqp->list);
1221                                         kfree(pqp);
1222                                 }
1223                                 kfree(entry);
1224                         }
1225                 }
1226         }
1227         kfree(priv->steer);
1228 }
1229
1230 static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
1231 {
1232         struct mlx4_priv *priv;
1233         struct mlx4_dev *dev;
1234         int err;
1235         int port;
1236
1237         pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev));
1238
1239         err = pci_enable_device(pdev);
1240         if (err) {
1241                 dev_err(&pdev->dev, "Cannot enable PCI device, "
1242                         "aborting.\n");
1243                 return err;
1244         }
1245
1246         /*
1247          * Check for BARs.  We expect 0: 1MB
1248          */
1249         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
1250             pci_resource_len(pdev, 0) != 1 << 20) {
1251                 dev_err(&pdev->dev, "Missing DCS, aborting.\n");
1252                 err = -ENODEV;
1253                 goto err_disable_pdev;
1254         }
1255         if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
1256                 dev_err(&pdev->dev, "Missing UAR, aborting.\n");
1257                 err = -ENODEV;
1258                 goto err_disable_pdev;
1259         }
1260
1261         err = pci_request_regions(pdev, DRV_NAME);
1262         if (err) {
1263                 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
1264                 goto err_disable_pdev;
1265         }
1266
1267         pci_set_master(pdev);
1268
1269         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1270         if (err) {
1271                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
1272                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1273                 if (err) {
1274                         dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
1275                         goto err_release_regions;
1276                 }
1277         }
1278         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1279         if (err) {
1280                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
1281                          "consistent PCI DMA mask.\n");
1282                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1283                 if (err) {
1284                         dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
1285                                 "aborting.\n");
1286                         goto err_release_regions;
1287                 }
1288         }
1289
1290         /* Allow large DMA segments, up to the firmware limit of 1 GB */
1291         dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024);
1292
1293         priv = kzalloc(sizeof *priv, GFP_KERNEL);
1294         if (!priv) {
1295                 dev_err(&pdev->dev, "Device struct alloc failed, "
1296                         "aborting.\n");
1297                 err = -ENOMEM;
1298                 goto err_release_regions;
1299         }
1300
1301         dev       = &priv->dev;
1302         dev->pdev = pdev;
1303         INIT_LIST_HEAD(&priv->ctx_list);
1304         spin_lock_init(&priv->ctx_lock);
1305
1306         mutex_init(&priv->port_mutex);
1307
1308         INIT_LIST_HEAD(&priv->pgdir_list);
1309         mutex_init(&priv->pgdir_mutex);
1310
1311         INIT_LIST_HEAD(&priv->bf_list);
1312         mutex_init(&priv->bf_mutex);
1313
1314         dev->rev_id = pdev->revision;
1315
1316         /*
1317          * Now reset the HCA before we touch the PCI capabilities or
1318          * attempt a firmware command, since a boot ROM may have left
1319          * the HCA in an undefined state.
1320          */
1321         err = mlx4_reset(dev);
1322         if (err) {
1323                 mlx4_err(dev, "Failed to reset HCA, aborting.\n");
1324                 goto err_free_dev;
1325         }
1326
1327         if (mlx4_cmd_init(dev)) {
1328                 mlx4_err(dev, "Failed to init command interface, aborting.\n");
1329                 goto err_free_dev;
1330         }
1331
1332         err = mlx4_init_hca(dev);
1333         if (err)
1334                 goto err_cmd;
1335
1336         err = mlx4_alloc_eq_table(dev);
1337         if (err)
1338                 goto err_close;
1339
1340         priv->msix_ctl.pool_bm = 0;
1341         spin_lock_init(&priv->msix_ctl.pool_lock);
1342
1343         mlx4_enable_msi_x(dev);
1344
1345         err = mlx4_init_steering(dev);
1346         if (err)
1347                 goto err_free_eq;
1348
1349         err = mlx4_setup_hca(dev);
1350         if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X)) {
1351                 dev->flags &= ~MLX4_FLAG_MSI_X;
1352                 pci_disable_msix(pdev);
1353                 err = mlx4_setup_hca(dev);
1354         }
1355
1356         if (err)
1357                 goto err_steer;
1358
1359         for (port = 1; port <= dev->caps.num_ports; port++) {
1360                 err = mlx4_init_port_info(dev, port);
1361                 if (err)
1362                         goto err_port;
1363         }
1364
1365         err = mlx4_register_device(dev);
1366         if (err)
1367                 goto err_port;
1368
1369         mlx4_sense_init(dev);
1370         mlx4_start_sense(dev);
1371
1372         pci_set_drvdata(pdev, dev);
1373
1374         return 0;
1375
1376 err_port:
1377         for (--port; port >= 1; --port)
1378                 mlx4_cleanup_port_info(&priv->port[port]);
1379
1380         mlx4_cleanup_counters_table(dev);
1381         mlx4_cleanup_mcg_table(dev);
1382         mlx4_cleanup_qp_table(dev);
1383         mlx4_cleanup_srq_table(dev);
1384         mlx4_cleanup_cq_table(dev);
1385         mlx4_cmd_use_polling(dev);
1386         mlx4_cleanup_eq_table(dev);
1387         mlx4_cleanup_mr_table(dev);
1388         mlx4_cleanup_xrcd_table(dev);
1389         mlx4_cleanup_pd_table(dev);
1390         mlx4_cleanup_uar_table(dev);
1391
1392 err_steer:
1393         mlx4_clear_steering(dev);
1394
1395 err_free_eq:
1396         mlx4_free_eq_table(dev);
1397
1398 err_close:
1399         if (dev->flags & MLX4_FLAG_MSI_X)
1400                 pci_disable_msix(pdev);
1401
1402         mlx4_close_hca(dev);
1403
1404 err_cmd:
1405         mlx4_cmd_cleanup(dev);
1406
1407 err_free_dev:
1408         kfree(priv);
1409
1410 err_release_regions:
1411         pci_release_regions(pdev);
1412
1413 err_disable_pdev:
1414         pci_disable_device(pdev);
1415         pci_set_drvdata(pdev, NULL);
1416         return err;
1417 }
1418
1419 static int __devinit mlx4_init_one(struct pci_dev *pdev,
1420                                    const struct pci_device_id *id)
1421 {
1422         printk_once(KERN_INFO "%s", mlx4_version);
1423
1424         return __mlx4_init_one(pdev, id);
1425 }
1426
1427 static void mlx4_remove_one(struct pci_dev *pdev)
1428 {
1429         struct mlx4_dev  *dev  = pci_get_drvdata(pdev);
1430         struct mlx4_priv *priv = mlx4_priv(dev);
1431         int p;
1432
1433         if (dev) {
1434                 mlx4_stop_sense(dev);
1435                 mlx4_unregister_device(dev);
1436
1437                 for (p = 1; p <= dev->caps.num_ports; p++) {
1438                         mlx4_cleanup_port_info(&priv->port[p]);
1439                         mlx4_CLOSE_PORT(dev, p);
1440                 }
1441
1442                 mlx4_cleanup_counters_table(dev);
1443                 mlx4_cleanup_mcg_table(dev);
1444                 mlx4_cleanup_qp_table(dev);
1445                 mlx4_cleanup_srq_table(dev);
1446                 mlx4_cleanup_cq_table(dev);
1447                 mlx4_cmd_use_polling(dev);
1448                 mlx4_cleanup_eq_table(dev);
1449                 mlx4_cleanup_mr_table(dev);
1450                 mlx4_cleanup_xrcd_table(dev);
1451                 mlx4_cleanup_pd_table(dev);
1452
1453                 iounmap(priv->kar);
1454                 mlx4_uar_free(dev, &priv->driver_uar);
1455                 mlx4_cleanup_uar_table(dev);
1456                 mlx4_clear_steering(dev);
1457                 mlx4_free_eq_table(dev);
1458                 mlx4_close_hca(dev);
1459                 mlx4_cmd_cleanup(dev);
1460
1461                 if (dev->flags & MLX4_FLAG_MSI_X)
1462                         pci_disable_msix(pdev);
1463
1464                 kfree(priv);
1465                 pci_release_regions(pdev);
1466                 pci_disable_device(pdev);
1467                 pci_set_drvdata(pdev, NULL);
1468         }
1469 }
1470
1471 int mlx4_restart_one(struct pci_dev *pdev)
1472 {
1473         mlx4_remove_one(pdev);
1474         return __mlx4_init_one(pdev, NULL);
1475 }
1476
1477 static DEFINE_PCI_DEVICE_TABLE(mlx4_pci_table) = {
1478         { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */
1479         { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */
1480         { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */
1481         { PCI_VDEVICE(MELLANOX, 0x6732) }, /* MT25408 "Hermon" DDR PCIe gen2 */
1482         { PCI_VDEVICE(MELLANOX, 0x673c) }, /* MT25408 "Hermon" QDR PCIe gen2 */
1483         { PCI_VDEVICE(MELLANOX, 0x6368) }, /* MT25408 "Hermon" EN 10GigE */
1484         { PCI_VDEVICE(MELLANOX, 0x6750) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */
1485         { PCI_VDEVICE(MELLANOX, 0x6372) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */
1486         { PCI_VDEVICE(MELLANOX, 0x675a) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
1487         { PCI_VDEVICE(MELLANOX, 0x6764) }, /* MT26468 ConnectX EN 10GigE PCIe gen2*/
1488         { PCI_VDEVICE(MELLANOX, 0x6746) }, /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
1489         { PCI_VDEVICE(MELLANOX, 0x676e) }, /* MT26478 ConnectX2 40GigE PCIe gen2 */
1490         { PCI_VDEVICE(MELLANOX, 0x1002) }, /* MT25400 Family [ConnectX-2 Virtual Function] */
1491         { PCI_VDEVICE(MELLANOX, 0x1003) }, /* MT27500 Family [ConnectX-3] */
1492         { PCI_VDEVICE(MELLANOX, 0x1004) }, /* MT27500 Family [ConnectX-3 Virtual Function] */
1493         { PCI_VDEVICE(MELLANOX, 0x1005) }, /* MT27510 Family */
1494         { PCI_VDEVICE(MELLANOX, 0x1006) }, /* MT27511 Family */
1495         { PCI_VDEVICE(MELLANOX, 0x1007) }, /* MT27520 Family */
1496         { PCI_VDEVICE(MELLANOX, 0x1008) }, /* MT27521 Family */
1497         { PCI_VDEVICE(MELLANOX, 0x1009) }, /* MT27530 Family */
1498         { PCI_VDEVICE(MELLANOX, 0x100a) }, /* MT27531 Family */
1499         { PCI_VDEVICE(MELLANOX, 0x100b) }, /* MT27540 Family */
1500         { PCI_VDEVICE(MELLANOX, 0x100c) }, /* MT27541 Family */
1501         { PCI_VDEVICE(MELLANOX, 0x100d) }, /* MT27550 Family */
1502         { PCI_VDEVICE(MELLANOX, 0x100e) }, /* MT27551 Family */
1503         { PCI_VDEVICE(MELLANOX, 0x100f) }, /* MT27560 Family */
1504         { PCI_VDEVICE(MELLANOX, 0x1010) }, /* MT27561 Family */
1505         { 0, }
1506 };
1507
1508 MODULE_DEVICE_TABLE(pci, mlx4_pci_table);
1509
1510 static struct pci_driver mlx4_driver = {
1511         .name           = DRV_NAME,
1512         .id_table       = mlx4_pci_table,
1513         .probe          = mlx4_init_one,
1514         .remove         = __devexit_p(mlx4_remove_one)
1515 };
1516
1517 static int __init mlx4_verify_params(void)
1518 {
1519         if ((log_num_mac < 0) || (log_num_mac > 7)) {
1520                 pr_warning("mlx4_core: bad num_mac: %d\n", log_num_mac);
1521                 return -1;
1522         }
1523
1524         if (log_num_vlan != 0)
1525                 pr_warning("mlx4_core: log_num_vlan - obsolete module param, using %d\n",
1526                            MLX4_LOG_NUM_VLANS);
1527
1528         if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 7)) {
1529                 pr_warning("mlx4_core: bad log_mtts_per_seg: %d\n", log_mtts_per_seg);
1530                 return -1;
1531         }
1532
1533         return 0;
1534 }
1535
1536 static int __init mlx4_init(void)
1537 {
1538         int ret;
1539
1540         if (mlx4_verify_params())
1541                 return -EINVAL;
1542
1543         mlx4_catas_init();
1544
1545         mlx4_wq = create_singlethread_workqueue("mlx4");
1546         if (!mlx4_wq)
1547                 return -ENOMEM;
1548
1549         ret = pci_register_driver(&mlx4_driver);
1550         return ret < 0 ? ret : 0;
1551 }
1552
1553 static void __exit mlx4_cleanup(void)
1554 {
1555         pci_unregister_driver(&mlx4_driver);
1556         destroy_workqueue(mlx4_wq);
1557 }
1558
1559 module_init(mlx4_init);
1560 module_exit(mlx4_cleanup);