]> www.infradead.org Git - users/dwmw2/linux.git/blob
1073741
[users/dwmw2/linux.git] /
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
3
4 #include <linux/etherdevice.h>
5 #include <linux/of_net.h>
6 #include <linux/pci.h>
7 #include <linux/bpf.h>
8 #include <generated/utsrelease.h>
9 #include <linux/crash_dump.h>
10
11 /* Local includes */
12 #include "i40e.h"
13 #include "i40e_diag.h"
14 #include "i40e_xsk.h"
15 #include <net/udp_tunnel.h>
16 #include <net/xdp_sock_drv.h>
17 /* All i40e tracepoints are defined by the include below, which
18  * must be included exactly once across the whole kernel with
19  * CREATE_TRACE_POINTS defined
20  */
21 #define CREATE_TRACE_POINTS
22 #include "i40e_trace.h"
23
24 const char i40e_driver_name[] = "i40e";
25 static const char i40e_driver_string[] =
26                         "Intel(R) Ethernet Connection XL710 Network Driver";
27
28 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
29
30 /* a bit of forward declarations */
31 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
32 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
33 static int i40e_add_vsi(struct i40e_vsi *vsi);
34 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
35 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
36 static int i40e_setup_misc_vector(struct i40e_pf *pf);
37 static void i40e_determine_queue_usage(struct i40e_pf *pf);
38 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
39 static void i40e_prep_for_reset(struct i40e_pf *pf);
40 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
41                                    bool lock_acquired);
42 static int i40e_reset(struct i40e_pf *pf);
43 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
44 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
45 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
46 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
47 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
48 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
49 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
50 static int i40e_get_capabilities(struct i40e_pf *pf,
51                                  enum i40e_admin_queue_opc list_type);
52 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
53
54 /* i40e_pci_tbl - PCI Device ID Table
55  *
56  * Last entry must be all 0s
57  *
58  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
59  *   Class, Class Mask, private data (not used) }
60  */
61 static const struct pci_device_id i40e_pci_tbl[] = {
62         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
63         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
64         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
86         /* required last entry */
87         {0, }
88 };
89 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
90
91 #define I40E_MAX_VF_COUNT 128
92 static int debug = -1;
93 module_param(debug, uint, 0);
94 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
95
96 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
97 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
98 MODULE_LICENSE("GPL v2");
99
100 static struct workqueue_struct *i40e_wq;
101
102 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
103                                   struct net_device *netdev, int delta)
104 {
105         struct netdev_hw_addr_list *ha_list;
106         struct netdev_hw_addr *ha;
107
108         if (!f || !netdev)
109                 return;
110
111         if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
112                 ha_list = &netdev->uc;
113         else
114                 ha_list = &netdev->mc;
115
116         netdev_hw_addr_list_for_each(ha, ha_list) {
117                 if (ether_addr_equal(ha->addr, f->macaddr)) {
118                         ha->refcount += delta;
119                         if (ha->refcount <= 0)
120                                 ha->refcount = 1;
121                         break;
122                 }
123         }
124 }
125
126 /**
127  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
128  * @hw:   pointer to the HW structure
129  * @mem:  ptr to mem struct to fill out
130  * @size: size of memory requested
131  * @alignment: what to align the allocation to
132  **/
133 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
134                             u64 size, u32 alignment)
135 {
136         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
137
138         mem->size = ALIGN(size, alignment);
139         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
140                                      GFP_KERNEL);
141         if (!mem->va)
142                 return -ENOMEM;
143
144         return 0;
145 }
146
147 /**
148  * i40e_free_dma_mem_d - OS specific memory free for shared code
149  * @hw:   pointer to the HW structure
150  * @mem:  ptr to mem struct to free
151  **/
152 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
153 {
154         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
155
156         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
157         mem->va = NULL;
158         mem->pa = 0;
159         mem->size = 0;
160
161         return 0;
162 }
163
164 /**
165  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
166  * @hw:   pointer to the HW structure
167  * @mem:  ptr to mem struct to fill out
168  * @size: size of memory requested
169  **/
170 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
171                              u32 size)
172 {
173         mem->size = size;
174         mem->va = kzalloc(size, GFP_KERNEL);
175
176         if (!mem->va)
177                 return -ENOMEM;
178
179         return 0;
180 }
181
182 /**
183  * i40e_free_virt_mem_d - OS specific memory free for shared code
184  * @hw:   pointer to the HW structure
185  * @mem:  ptr to mem struct to free
186  **/
187 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
188 {
189         /* it's ok to kfree a NULL pointer */
190         kfree(mem->va);
191         mem->va = NULL;
192         mem->size = 0;
193
194         return 0;
195 }
196
197 /**
198  * i40e_get_lump - find a lump of free generic resource
199  * @pf: board private structure
200  * @pile: the pile of resource to search
201  * @needed: the number of items needed
202  * @id: an owner id to stick on the items assigned
203  *
204  * Returns the base item index of the lump, or negative for error
205  **/
206 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
207                          u16 needed, u16 id)
208 {
209         int ret = -ENOMEM;
210         int i, j;
211
212         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
213                 dev_info(&pf->pdev->dev,
214                          "param err: pile=%s needed=%d id=0x%04x\n",
215                          pile ? "<valid>" : "<null>", needed, id);
216                 return -EINVAL;
217         }
218
219         /* Allocate last queue in the pile for FDIR VSI queue
220          * so it doesn't fragment the qp_pile
221          */
222         if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
223                 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
224                         dev_err(&pf->pdev->dev,
225                                 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
226                                 pile->num_entries - 1);
227                         return -ENOMEM;
228                 }
229                 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
230                 return pile->num_entries - 1;
231         }
232
233         i = 0;
234         while (i < pile->num_entries) {
235                 /* skip already allocated entries */
236                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
237                         i++;
238                         continue;
239                 }
240
241                 /* do we have enough in this lump? */
242                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
243                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
244                                 break;
245                 }
246
247                 if (j == needed) {
248                         /* there was enough, so assign it to the requestor */
249                         for (j = 0; j < needed; j++)
250                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
251                         ret = i;
252                         break;
253                 }
254
255                 /* not enough, so skip over it and continue looking */
256                 i += j;
257         }
258
259         return ret;
260 }
261
262 /**
263  * i40e_put_lump - return a lump of generic resource
264  * @pile: the pile of resource to search
265  * @index: the base item index
266  * @id: the owner id of the items assigned
267  *
268  * Returns the count of items in the lump
269  **/
270 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
271 {
272         int valid_id = (id | I40E_PILE_VALID_BIT);
273         int count = 0;
274         u16 i;
275
276         if (!pile || index >= pile->num_entries)
277                 return -EINVAL;
278
279         for (i = index;
280              i < pile->num_entries && pile->list[i] == valid_id;
281              i++) {
282                 pile->list[i] = 0;
283                 count++;
284         }
285
286
287         return count;
288 }
289
290 /**
291  * i40e_find_vsi_from_id - searches for the vsi with the given id
292  * @pf: the pf structure to search for the vsi
293  * @id: id of the vsi it is searching for
294  **/
295 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
296 {
297         int i;
298
299         for (i = 0; i < pf->num_alloc_vsi; i++)
300                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
301                         return pf->vsi[i];
302
303         return NULL;
304 }
305
306 /**
307  * i40e_service_event_schedule - Schedule the service task to wake up
308  * @pf: board private structure
309  *
310  * If not already scheduled, this puts the task into the work queue
311  **/
312 void i40e_service_event_schedule(struct i40e_pf *pf)
313 {
314         if ((!test_bit(__I40E_DOWN, pf->state) &&
315              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
316               test_bit(__I40E_RECOVERY_MODE, pf->state))
317                 queue_work(i40e_wq, &pf->service_task);
318 }
319
320 /**
321  * i40e_tx_timeout - Respond to a Tx Hang
322  * @netdev: network interface device structure
323  * @txqueue: queue number timing out
324  *
325  * If any port has noticed a Tx timeout, it is likely that the whole
326  * device is munged, not just the one netdev port, so go for the full
327  * reset.
328  **/
329 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
330 {
331         struct i40e_netdev_priv *np = netdev_priv(netdev);
332         struct i40e_vsi *vsi = np->vsi;
333         struct i40e_pf *pf = vsi->back;
334         struct i40e_ring *tx_ring = NULL;
335         unsigned int i;
336         u32 head, val;
337
338         pf->tx_timeout_count++;
339
340         /* with txqueue index, find the tx_ring struct */
341         for (i = 0; i < vsi->num_queue_pairs; i++) {
342                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
343                         if (txqueue ==
344                             vsi->tx_rings[i]->queue_index) {
345                                 tx_ring = vsi->tx_rings[i];
346                                 break;
347                         }
348                 }
349         }
350
351         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
352                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
353         else if (time_before(jiffies,
354                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
355                 return;   /* don't do any new action before the next timeout */
356
357         /* don't kick off another recovery if one is already pending */
358         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
359                 return;
360
361         if (tx_ring) {
362                 head = i40e_get_head(tx_ring);
363                 /* Read interrupt register */
364                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
365                         val = rd32(&pf->hw,
366                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
367                                                 tx_ring->vsi->base_vector - 1));
368                 else
369                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
370
371                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
372                             vsi->seid, txqueue, tx_ring->next_to_clean,
373                             head, tx_ring->next_to_use,
374                             readl(tx_ring->tail), val);
375         }
376
377         pf->tx_timeout_last_recovery = jiffies;
378         netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
379                     pf->tx_timeout_recovery_level, txqueue);
380
381         switch (pf->tx_timeout_recovery_level) {
382         case 1:
383                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
384                 break;
385         case 2:
386                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
387                 break;
388         case 3:
389                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
390                 break;
391         default:
392                 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
393                 set_bit(__I40E_DOWN_REQUESTED, pf->state);
394                 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
395                 break;
396         }
397
398         i40e_service_event_schedule(pf);
399         pf->tx_timeout_recovery_level++;
400 }
401
402 /**
403  * i40e_get_vsi_stats_struct - Get System Network Statistics
404  * @vsi: the VSI we care about
405  *
406  * Returns the address of the device statistics structure.
407  * The statistics are actually updated from the service task.
408  **/
409 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
410 {
411         return &vsi->net_stats;
412 }
413
414 /**
415  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
416  * @ring: Tx ring to get statistics from
417  * @stats: statistics entry to be updated
418  **/
419 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
420                                             struct rtnl_link_stats64 *stats)
421 {
422         u64 bytes, packets;
423         unsigned int start;
424
425         do {
426                 start = u64_stats_fetch_begin_irq(&ring->syncp);
427                 packets = ring->stats.packets;
428                 bytes   = ring->stats.bytes;
429         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
430
431         stats->tx_packets += packets;
432         stats->tx_bytes   += bytes;
433 }
434
435 /**
436  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
437  * @netdev: network interface device structure
438  * @stats: data structure to store statistics
439  *
440  * Returns the address of the device statistics structure.
441  * The statistics are actually updated from the service task.
442  **/
443 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
444                                   struct rtnl_link_stats64 *stats)
445 {
446         struct i40e_netdev_priv *np = netdev_priv(netdev);
447         struct i40e_vsi *vsi = np->vsi;
448         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
449         struct i40e_ring *ring;
450         int i;
451
452         if (test_bit(__I40E_VSI_DOWN, vsi->state))
453                 return;
454
455         if (!vsi->tx_rings)
456                 return;
457
458         rcu_read_lock();
459         for (i = 0; i < vsi->num_queue_pairs; i++) {
460                 u64 bytes, packets;
461                 unsigned int start;
462
463                 ring = READ_ONCE(vsi->tx_rings[i]);
464                 if (!ring)
465                         continue;
466                 i40e_get_netdev_stats_struct_tx(ring, stats);
467
468                 if (i40e_enabled_xdp_vsi(vsi)) {
469                         ring = READ_ONCE(vsi->xdp_rings[i]);
470                         if (!ring)
471                                 continue;
472                         i40e_get_netdev_stats_struct_tx(ring, stats);
473                 }
474
475                 ring = READ_ONCE(vsi->rx_rings[i]);
476                 if (!ring)
477                         continue;
478                 do {
479                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
480                         packets = ring->stats.packets;
481                         bytes   = ring->stats.bytes;
482                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
483
484                 stats->rx_packets += packets;
485                 stats->rx_bytes   += bytes;
486
487         }
488         rcu_read_unlock();
489
490         /* following stats updated by i40e_watchdog_subtask() */
491         stats->multicast        = vsi_stats->multicast;
492         stats->tx_errors        = vsi_stats->tx_errors;
493         stats->tx_dropped       = vsi_stats->tx_dropped;
494         stats->rx_errors        = vsi_stats->rx_errors;
495         stats->rx_dropped       = vsi_stats->rx_dropped;
496         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
497         stats->rx_length_errors = vsi_stats->rx_length_errors;
498 }
499
500 /**
501  * i40e_vsi_reset_stats - Resets all stats of the given vsi
502  * @vsi: the VSI to have its stats reset
503  **/
504 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
505 {
506         struct rtnl_link_stats64 *ns;
507         int i;
508
509         if (!vsi)
510                 return;
511
512         ns = i40e_get_vsi_stats_struct(vsi);
513         memset(ns, 0, sizeof(*ns));
514         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
515         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
516         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
517         if (vsi->rx_rings && vsi->rx_rings[0]) {
518                 for (i = 0; i < vsi->num_queue_pairs; i++) {
519                         memset(&vsi->rx_rings[i]->stats, 0,
520                                sizeof(vsi->rx_rings[i]->stats));
521                         memset(&vsi->rx_rings[i]->rx_stats, 0,
522                                sizeof(vsi->rx_rings[i]->rx_stats));
523                         memset(&vsi->tx_rings[i]->stats, 0,
524                                sizeof(vsi->tx_rings[i]->stats));
525                         memset(&vsi->tx_rings[i]->tx_stats, 0,
526                                sizeof(vsi->tx_rings[i]->tx_stats));
527                 }
528         }
529         vsi->stat_offsets_loaded = false;
530 }
531
532 /**
533  * i40e_pf_reset_stats - Reset all of the stats for the given PF
534  * @pf: the PF to be reset
535  **/
536 void i40e_pf_reset_stats(struct i40e_pf *pf)
537 {
538         int i;
539
540         memset(&pf->stats, 0, sizeof(pf->stats));
541         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
542         pf->stat_offsets_loaded = false;
543
544         for (i = 0; i < I40E_MAX_VEB; i++) {
545                 if (pf->veb[i]) {
546                         memset(&pf->veb[i]->stats, 0,
547                                sizeof(pf->veb[i]->stats));
548                         memset(&pf->veb[i]->stats_offsets, 0,
549                                sizeof(pf->veb[i]->stats_offsets));
550                         memset(&pf->veb[i]->tc_stats, 0,
551                                sizeof(pf->veb[i]->tc_stats));
552                         memset(&pf->veb[i]->tc_stats_offsets, 0,
553                                sizeof(pf->veb[i]->tc_stats_offsets));
554                         pf->veb[i]->stat_offsets_loaded = false;
555                 }
556         }
557         pf->hw_csum_rx_error = 0;
558 }
559
560 /**
561  * i40e_compute_pci_to_hw_id - compute index form PCI function.
562  * @vsi: ptr to the VSI to read from.
563  * @hw: ptr to the hardware info.
564  **/
565 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
566 {
567         int pf_count = i40e_get_pf_count(hw);
568
569         if (vsi->type == I40E_VSI_SRIOV)
570                 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
571
572         return hw->port + BIT(7);
573 }
574
575 /**
576  * i40e_stat_update64 - read and update a 64 bit stat from the chip.
577  * @hw: ptr to the hardware info.
578  * @hireg: the high 32 bit reg to read.
579  * @loreg: the low 32 bit reg to read.
580  * @offset_loaded: has the initial offset been loaded yet.
581  * @offset: ptr to current offset value.
582  * @stat: ptr to the stat.
583  *
584  * Since the device stats are not reset at PFReset, they will not
585  * be zeroed when the driver starts.  We'll save the first values read
586  * and use them as offsets to be subtracted from the raw values in order
587  * to report stats that count from zero.
588  **/
589 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
590                                bool offset_loaded, u64 *offset, u64 *stat)
591 {
592         u64 new_data;
593
594         new_data = rd64(hw, loreg);
595
596         if (!offset_loaded || new_data < *offset)
597                 *offset = new_data;
598         *stat = new_data - *offset;
599 }
600
601 /**
602  * i40e_stat_update48 - read and update a 48 bit stat from the chip
603  * @hw: ptr to the hardware info
604  * @hireg: the high 32 bit reg to read
605  * @loreg: the low 32 bit reg to read
606  * @offset_loaded: has the initial offset been loaded yet
607  * @offset: ptr to current offset value
608  * @stat: ptr to the stat
609  *
610  * Since the device stats are not reset at PFReset, they likely will not
611  * be zeroed when the driver starts.  We'll save the first values read
612  * and use them as offsets to be subtracted from the raw values in order
613  * to report stats that count from zero.  In the process, we also manage
614  * the potential roll-over.
615  **/
616 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
617                                bool offset_loaded, u64 *offset, u64 *stat)
618 {
619         u64 new_data;
620
621         if (hw->device_id == I40E_DEV_ID_QEMU) {
622                 new_data = rd32(hw, loreg);
623                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
624         } else {
625                 new_data = rd64(hw, loreg);
626         }
627         if (!offset_loaded)
628                 *offset = new_data;
629         if (likely(new_data >= *offset))
630                 *stat = new_data - *offset;
631         else
632                 *stat = (new_data + BIT_ULL(48)) - *offset;
633         *stat &= 0xFFFFFFFFFFFFULL;
634 }
635
636 /**
637  * i40e_stat_update32 - read and update a 32 bit stat from the chip
638  * @hw: ptr to the hardware info
639  * @reg: the hw reg to read
640  * @offset_loaded: has the initial offset been loaded yet
641  * @offset: ptr to current offset value
642  * @stat: ptr to the stat
643  **/
644 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
645                                bool offset_loaded, u64 *offset, u64 *stat)
646 {
647         u32 new_data;
648
649         new_data = rd32(hw, reg);
650         if (!offset_loaded)
651                 *offset = new_data;
652         if (likely(new_data >= *offset))
653                 *stat = (u32)(new_data - *offset);
654         else
655                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
656 }
657
658 /**
659  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
660  * @hw: ptr to the hardware info
661  * @reg: the hw reg to read and clear
662  * @stat: ptr to the stat
663  **/
664 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
665 {
666         u32 new_data = rd32(hw, reg);
667
668         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
669         *stat += new_data;
670 }
671
672 /**
673  * i40e_stats_update_rx_discards - update rx_discards.
674  * @vsi: ptr to the VSI to be updated.
675  * @hw: ptr to the hardware info.
676  * @stat_idx: VSI's stat_counter_idx.
677  * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
678  * @stat_offset: ptr to stat_offset to store first read of specific register.
679  * @stat: ptr to VSI's stat to be updated.
680  **/
681 static void
682 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
683                               int stat_idx, bool offset_loaded,
684                               struct i40e_eth_stats *stat_offset,
685                               struct i40e_eth_stats *stat)
686 {
687         u64 rx_rdpc, rx_rxerr;
688
689         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
690                            &stat_offset->rx_discards, &rx_rdpc);
691         i40e_stat_update64(hw,
692                            I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
693                            I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
694                            offset_loaded, &stat_offset->rx_discards_other,
695                            &rx_rxerr);
696
697         stat->rx_discards = rx_rdpc + rx_rxerr;
698 }
699
700 /**
701  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
702  * @vsi: the VSI to be updated
703  **/
704 void i40e_update_eth_stats(struct i40e_vsi *vsi)
705 {
706         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
707         struct i40e_pf *pf = vsi->back;
708         struct i40e_hw *hw = &pf->hw;
709         struct i40e_eth_stats *oes;
710         struct i40e_eth_stats *es;     /* device's eth stats */
711
712         es = &vsi->eth_stats;
713         oes = &vsi->eth_stats_offsets;
714
715         /* Gather up the stats that the hw collects */
716         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
717                            vsi->stat_offsets_loaded,
718                            &oes->tx_errors, &es->tx_errors);
719         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
720                            vsi->stat_offsets_loaded,
721                            &oes->rx_discards, &es->rx_discards);
722         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
723                            vsi->stat_offsets_loaded,
724                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
725
726         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
727                            I40E_GLV_GORCL(stat_idx),
728                            vsi->stat_offsets_loaded,
729                            &oes->rx_bytes, &es->rx_bytes);
730         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
731                            I40E_GLV_UPRCL(stat_idx),
732                            vsi->stat_offsets_loaded,
733                            &oes->rx_unicast, &es->rx_unicast);
734         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
735                            I40E_GLV_MPRCL(stat_idx),
736                            vsi->stat_offsets_loaded,
737                            &oes->rx_multicast, &es->rx_multicast);
738         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
739                            I40E_GLV_BPRCL(stat_idx),
740                            vsi->stat_offsets_loaded,
741                            &oes->rx_broadcast, &es->rx_broadcast);
742
743         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
744                            I40E_GLV_GOTCL(stat_idx),
745                            vsi->stat_offsets_loaded,
746                            &oes->tx_bytes, &es->tx_bytes);
747         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
748                            I40E_GLV_UPTCL(stat_idx),
749                            vsi->stat_offsets_loaded,
750                            &oes->tx_unicast, &es->tx_unicast);
751         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
752                            I40E_GLV_MPTCL(stat_idx),
753                            vsi->stat_offsets_loaded,
754                            &oes->tx_multicast, &es->tx_multicast);
755         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
756                            I40E_GLV_BPTCL(stat_idx),
757                            vsi->stat_offsets_loaded,
758                            &oes->tx_broadcast, &es->tx_broadcast);
759
760         i40e_stats_update_rx_discards(vsi, hw, stat_idx,
761                                       vsi->stat_offsets_loaded, oes, es);
762
763         vsi->stat_offsets_loaded = true;
764 }
765
766 /**
767  * i40e_update_veb_stats - Update Switch component statistics
768  * @veb: the VEB being updated
769  **/
770 void i40e_update_veb_stats(struct i40e_veb *veb)
771 {
772         struct i40e_pf *pf = veb->pf;
773         struct i40e_hw *hw = &pf->hw;
774         struct i40e_eth_stats *oes;
775         struct i40e_eth_stats *es;     /* device's eth stats */
776         struct i40e_veb_tc_stats *veb_oes;
777         struct i40e_veb_tc_stats *veb_es;
778         int i, idx = 0;
779
780         idx = veb->stats_idx;
781         es = &veb->stats;
782         oes = &veb->stats_offsets;
783         veb_es = &veb->tc_stats;
784         veb_oes = &veb->tc_stats_offsets;
785
786         /* Gather up the stats that the hw collects */
787         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
788                            veb->stat_offsets_loaded,
789                            &oes->tx_discards, &es->tx_discards);
790         if (hw->revision_id > 0)
791                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
792                                    veb->stat_offsets_loaded,
793                                    &oes->rx_unknown_protocol,
794                                    &es->rx_unknown_protocol);
795         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
796                            veb->stat_offsets_loaded,
797                            &oes->rx_bytes, &es->rx_bytes);
798         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
799                            veb->stat_offsets_loaded,
800                            &oes->rx_unicast, &es->rx_unicast);
801         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
802                            veb->stat_offsets_loaded,
803                            &oes->rx_multicast, &es->rx_multicast);
804         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
805                            veb->stat_offsets_loaded,
806                            &oes->rx_broadcast, &es->rx_broadcast);
807
808         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
809                            veb->stat_offsets_loaded,
810                            &oes->tx_bytes, &es->tx_bytes);
811         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
812                            veb->stat_offsets_loaded,
813                            &oes->tx_unicast, &es->tx_unicast);
814         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
815                            veb->stat_offsets_loaded,
816                            &oes->tx_multicast, &es->tx_multicast);
817         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
818                            veb->stat_offsets_loaded,
819                            &oes->tx_broadcast, &es->tx_broadcast);
820         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
821                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
822                                    I40E_GLVEBTC_RPCL(i, idx),
823                                    veb->stat_offsets_loaded,
824                                    &veb_oes->tc_rx_packets[i],
825                                    &veb_es->tc_rx_packets[i]);
826                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
827                                    I40E_GLVEBTC_RBCL(i, idx),
828                                    veb->stat_offsets_loaded,
829                                    &veb_oes->tc_rx_bytes[i],
830                                    &veb_es->tc_rx_bytes[i]);
831                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
832                                    I40E_GLVEBTC_TPCL(i, idx),
833                                    veb->stat_offsets_loaded,
834                                    &veb_oes->tc_tx_packets[i],
835                                    &veb_es->tc_tx_packets[i]);
836                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
837                                    I40E_GLVEBTC_TBCL(i, idx),
838                                    veb->stat_offsets_loaded,
839                                    &veb_oes->tc_tx_bytes[i],
840                                    &veb_es->tc_tx_bytes[i]);
841         }
842         veb->stat_offsets_loaded = true;
843 }
844
845 /**
846  * i40e_update_vsi_stats - Update the vsi statistics counters.
847  * @vsi: the VSI to be updated
848  *
849  * There are a few instances where we store the same stat in a
850  * couple of different structs.  This is partly because we have
851  * the netdev stats that need to be filled out, which is slightly
852  * different from the "eth_stats" defined by the chip and used in
853  * VF communications.  We sort it out here.
854  **/
855 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
856 {
857         struct i40e_pf *pf = vsi->back;
858         struct rtnl_link_stats64 *ons;
859         struct rtnl_link_stats64 *ns;   /* netdev stats */
860         struct i40e_eth_stats *oes;
861         struct i40e_eth_stats *es;     /* device's eth stats */
862         u64 tx_restart, tx_busy;
863         struct i40e_ring *p;
864         u64 rx_page, rx_buf;
865         u64 bytes, packets;
866         unsigned int start;
867         u64 tx_linearize;
868         u64 tx_force_wb;
869         u64 rx_p, rx_b;
870         u64 tx_p, tx_b;
871         u16 q;
872
873         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
874             test_bit(__I40E_CONFIG_BUSY, pf->state))
875                 return;
876
877         ns = i40e_get_vsi_stats_struct(vsi);
878         ons = &vsi->net_stats_offsets;
879         es = &vsi->eth_stats;
880         oes = &vsi->eth_stats_offsets;
881
882         /* Gather up the netdev and vsi stats that the driver collects
883          * on the fly during packet processing
884          */
885         rx_b = rx_p = 0;
886         tx_b = tx_p = 0;
887         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
888         rx_page = 0;
889         rx_buf = 0;
890         rcu_read_lock();
891         for (q = 0; q < vsi->num_queue_pairs; q++) {
892                 /* locate Tx ring */
893                 p = READ_ONCE(vsi->tx_rings[q]);
894                 if (!p)
895                         continue;
896
897                 do {
898                         start = u64_stats_fetch_begin_irq(&p->syncp);
899                         packets = p->stats.packets;
900                         bytes = p->stats.bytes;
901                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
902                 tx_b += bytes;
903                 tx_p += packets;
904                 tx_restart += p->tx_stats.restart_queue;
905                 tx_busy += p->tx_stats.tx_busy;
906                 tx_linearize += p->tx_stats.tx_linearize;
907                 tx_force_wb += p->tx_stats.tx_force_wb;
908
909                 /* locate Rx ring */
910                 p = READ_ONCE(vsi->rx_rings[q]);
911                 if (!p)
912                         continue;
913
914                 do {
915                         start = u64_stats_fetch_begin_irq(&p->syncp);
916                         packets = p->stats.packets;
917                         bytes = p->stats.bytes;
918                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
919                 rx_b += bytes;
920                 rx_p += packets;
921                 rx_buf += p->rx_stats.alloc_buff_failed;
922                 rx_page += p->rx_stats.alloc_page_failed;
923
924                 if (i40e_enabled_xdp_vsi(vsi)) {
925                         /* locate XDP ring */
926                         p = READ_ONCE(vsi->xdp_rings[q]);
927                         if (!p)
928                                 continue;
929
930                         do {
931                                 start = u64_stats_fetch_begin_irq(&p->syncp);
932                                 packets = p->stats.packets;
933                                 bytes = p->stats.bytes;
934                         } while (u64_stats_fetch_retry_irq(&p->syncp, start));
935                         tx_b += bytes;
936                         tx_p += packets;
937                         tx_restart += p->tx_stats.restart_queue;
938                         tx_busy += p->tx_stats.tx_busy;
939                         tx_linearize += p->tx_stats.tx_linearize;
940                         tx_force_wb += p->tx_stats.tx_force_wb;
941                 }
942         }
943         rcu_read_unlock();
944         vsi->tx_restart = tx_restart;
945         vsi->tx_busy = tx_busy;
946         vsi->tx_linearize = tx_linearize;
947         vsi->tx_force_wb = tx_force_wb;
948         vsi->rx_page_failed = rx_page;
949         vsi->rx_buf_failed = rx_buf;
950
951         ns->rx_packets = rx_p;
952         ns->rx_bytes = rx_b;
953         ns->tx_packets = tx_p;
954         ns->tx_bytes = tx_b;
955
956         /* update netdev stats from eth stats */
957         i40e_update_eth_stats(vsi);
958         ons->tx_errors = oes->tx_errors;
959         ns->tx_errors = es->tx_errors;
960         ons->multicast = oes->rx_multicast;
961         ns->multicast = es->rx_multicast;
962         ons->rx_dropped = oes->rx_discards;
963         ns->rx_dropped = es->rx_discards;
964         ons->tx_dropped = oes->tx_discards;
965         ns->tx_dropped = es->tx_discards;
966
967         /* pull in a couple PF stats if this is the main vsi */
968         if (vsi == pf->vsi[pf->lan_vsi]) {
969                 ns->rx_crc_errors = pf->stats.crc_errors;
970                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
971                 ns->rx_length_errors = pf->stats.rx_length_errors;
972         }
973 }
974
975 /**
976  * i40e_update_pf_stats - Update the PF statistics counters.
977  * @pf: the PF to be updated
978  **/
979 static void i40e_update_pf_stats(struct i40e_pf *pf)
980 {
981         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
982         struct i40e_hw_port_stats *nsd = &pf->stats;
983         struct i40e_hw *hw = &pf->hw;
984         u32 val;
985         int i;
986
987         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
988                            I40E_GLPRT_GORCL(hw->port),
989                            pf->stat_offsets_loaded,
990                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
991         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
992                            I40E_GLPRT_GOTCL(hw->port),
993                            pf->stat_offsets_loaded,
994                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
995         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
996                            pf->stat_offsets_loaded,
997                            &osd->eth.rx_discards,
998                            &nsd->eth.rx_discards);
999         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1000                            I40E_GLPRT_UPRCL(hw->port),
1001                            pf->stat_offsets_loaded,
1002                            &osd->eth.rx_unicast,
1003                            &nsd->eth.rx_unicast);
1004         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1005                            I40E_GLPRT_MPRCL(hw->port),
1006                            pf->stat_offsets_loaded,
1007                            &osd->eth.rx_multicast,
1008                            &nsd->eth.rx_multicast);
1009         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1010                            I40E_GLPRT_BPRCL(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->eth.rx_broadcast,
1013                            &nsd->eth.rx_broadcast);
1014         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1015                            I40E_GLPRT_UPTCL(hw->port),
1016                            pf->stat_offsets_loaded,
1017                            &osd->eth.tx_unicast,
1018                            &nsd->eth.tx_unicast);
1019         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1020                            I40E_GLPRT_MPTCL(hw->port),
1021                            pf->stat_offsets_loaded,
1022                            &osd->eth.tx_multicast,
1023                            &nsd->eth.tx_multicast);
1024         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1025                            I40E_GLPRT_BPTCL(hw->port),
1026                            pf->stat_offsets_loaded,
1027                            &osd->eth.tx_broadcast,
1028                            &nsd->eth.tx_broadcast);
1029
1030         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1031                            pf->stat_offsets_loaded,
1032                            &osd->tx_dropped_link_down,
1033                            &nsd->tx_dropped_link_down);
1034
1035         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->crc_errors, &nsd->crc_errors);
1038
1039         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->illegal_bytes, &nsd->illegal_bytes);
1042
1043         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->mac_local_faults,
1046                            &nsd->mac_local_faults);
1047         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->mac_remote_faults,
1050                            &nsd->mac_remote_faults);
1051
1052         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_length_errors,
1055                            &nsd->rx_length_errors);
1056
1057         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1058                            pf->stat_offsets_loaded,
1059                            &osd->link_xon_rx, &nsd->link_xon_rx);
1060         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1061                            pf->stat_offsets_loaded,
1062                            &osd->link_xon_tx, &nsd->link_xon_tx);
1063         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1064                            pf->stat_offsets_loaded,
1065                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
1066         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1067                            pf->stat_offsets_loaded,
1068                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
1069
1070         for (i = 0; i < 8; i++) {
1071                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1072                                    pf->stat_offsets_loaded,
1073                                    &osd->priority_xoff_rx[i],
1074                                    &nsd->priority_xoff_rx[i]);
1075                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1076                                    pf->stat_offsets_loaded,
1077                                    &osd->priority_xon_rx[i],
1078                                    &nsd->priority_xon_rx[i]);
1079                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1080                                    pf->stat_offsets_loaded,
1081                                    &osd->priority_xon_tx[i],
1082                                    &nsd->priority_xon_tx[i]);
1083                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1084                                    pf->stat_offsets_loaded,
1085                                    &osd->priority_xoff_tx[i],
1086                                    &nsd->priority_xoff_tx[i]);
1087                 i40e_stat_update32(hw,
1088                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1089                                    pf->stat_offsets_loaded,
1090                                    &osd->priority_xon_2_xoff[i],
1091                                    &nsd->priority_xon_2_xoff[i]);
1092         }
1093
1094         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1095                            I40E_GLPRT_PRC64L(hw->port),
1096                            pf->stat_offsets_loaded,
1097                            &osd->rx_size_64, &nsd->rx_size_64);
1098         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1099                            I40E_GLPRT_PRC127L(hw->port),
1100                            pf->stat_offsets_loaded,
1101                            &osd->rx_size_127, &nsd->rx_size_127);
1102         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1103                            I40E_GLPRT_PRC255L(hw->port),
1104                            pf->stat_offsets_loaded,
1105                            &osd->rx_size_255, &nsd->rx_size_255);
1106         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1107                            I40E_GLPRT_PRC511L(hw->port),
1108                            pf->stat_offsets_loaded,
1109                            &osd->rx_size_511, &nsd->rx_size_511);
1110         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1111                            I40E_GLPRT_PRC1023L(hw->port),
1112                            pf->stat_offsets_loaded,
1113                            &osd->rx_size_1023, &nsd->rx_size_1023);
1114         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1115                            I40E_GLPRT_PRC1522L(hw->port),
1116                            pf->stat_offsets_loaded,
1117                            &osd->rx_size_1522, &nsd->rx_size_1522);
1118         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1119                            I40E_GLPRT_PRC9522L(hw->port),
1120                            pf->stat_offsets_loaded,
1121                            &osd->rx_size_big, &nsd->rx_size_big);
1122
1123         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1124                            I40E_GLPRT_PTC64L(hw->port),
1125                            pf->stat_offsets_loaded,
1126                            &osd->tx_size_64, &nsd->tx_size_64);
1127         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1128                            I40E_GLPRT_PTC127L(hw->port),
1129                            pf->stat_offsets_loaded,
1130                            &osd->tx_size_127, &nsd->tx_size_127);
1131         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1132                            I40E_GLPRT_PTC255L(hw->port),
1133                            pf->stat_offsets_loaded,
1134                            &osd->tx_size_255, &nsd->tx_size_255);
1135         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1136                            I40E_GLPRT_PTC511L(hw->port),
1137                            pf->stat_offsets_loaded,
1138                            &osd->tx_size_511, &nsd->tx_size_511);
1139         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1140                            I40E_GLPRT_PTC1023L(hw->port),
1141                            pf->stat_offsets_loaded,
1142                            &osd->tx_size_1023, &nsd->tx_size_1023);
1143         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1144                            I40E_GLPRT_PTC1522L(hw->port),
1145                            pf->stat_offsets_loaded,
1146                            &osd->tx_size_1522, &nsd->tx_size_1522);
1147         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1148                            I40E_GLPRT_PTC9522L(hw->port),
1149                            pf->stat_offsets_loaded,
1150                            &osd->tx_size_big, &nsd->tx_size_big);
1151
1152         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1153                            pf->stat_offsets_loaded,
1154                            &osd->rx_undersize, &nsd->rx_undersize);
1155         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1156                            pf->stat_offsets_loaded,
1157                            &osd->rx_fragments, &nsd->rx_fragments);
1158         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1159                            pf->stat_offsets_loaded,
1160                            &osd->rx_oversize, &nsd->rx_oversize);
1161         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1162                            pf->stat_offsets_loaded,
1163                            &osd->rx_jabber, &nsd->rx_jabber);
1164
1165         /* FDIR stats */
1166         i40e_stat_update_and_clear32(hw,
1167                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1168                         &nsd->fd_atr_match);
1169         i40e_stat_update_and_clear32(hw,
1170                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1171                         &nsd->fd_sb_match);
1172         i40e_stat_update_and_clear32(hw,
1173                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1174                         &nsd->fd_atr_tunnel_match);
1175
1176         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1177         nsd->tx_lpi_status =
1178                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1179                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1180         nsd->rx_lpi_status =
1181                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1182                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1183         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1184                            pf->stat_offsets_loaded,
1185                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1186         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1187                            pf->stat_offsets_loaded,
1188                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1189
1190         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1191             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1192                 nsd->fd_sb_status = true;
1193         else
1194                 nsd->fd_sb_status = false;
1195
1196         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1197             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1198                 nsd->fd_atr_status = true;
1199         else
1200                 nsd->fd_atr_status = false;
1201
1202         pf->stat_offsets_loaded = true;
1203 }
1204
1205 /**
1206  * i40e_update_stats - Update the various statistics counters.
1207  * @vsi: the VSI to be updated
1208  *
1209  * Update the various stats for this VSI and its related entities.
1210  **/
1211 void i40e_update_stats(struct i40e_vsi *vsi)
1212 {
1213         struct i40e_pf *pf = vsi->back;
1214
1215         if (vsi == pf->vsi[pf->lan_vsi])
1216                 i40e_update_pf_stats(pf);
1217
1218         i40e_update_vsi_stats(vsi);
1219 }
1220
1221 /**
1222  * i40e_count_filters - counts VSI mac filters
1223  * @vsi: the VSI to be searched
1224  *
1225  * Returns count of mac filters
1226  **/
1227 int i40e_count_filters(struct i40e_vsi *vsi)
1228 {
1229         struct i40e_mac_filter *f;
1230         struct hlist_node *h;
1231         int bkt;
1232         int cnt = 0;
1233
1234         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1235                 ++cnt;
1236
1237         return cnt;
1238 }
1239
1240 /**
1241  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1242  * @vsi: the VSI to be searched
1243  * @macaddr: the MAC address
1244  * @vlan: the vlan
1245  *
1246  * Returns ptr to the filter object or NULL
1247  **/
1248 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1249                                                 const u8 *macaddr, s16 vlan)
1250 {
1251         struct i40e_mac_filter *f;
1252         u64 key;
1253
1254         if (!vsi || !macaddr)
1255                 return NULL;
1256
1257         key = i40e_addr_to_hkey(macaddr);
1258         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1259                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1260                     (vlan == f->vlan))
1261                         return f;
1262         }
1263         return NULL;
1264 }
1265
1266 /**
1267  * i40e_find_mac - Find a mac addr in the macvlan filters list
1268  * @vsi: the VSI to be searched
1269  * @macaddr: the MAC address we are searching for
1270  *
1271  * Returns the first filter with the provided MAC address or NULL if
1272  * MAC address was not found
1273  **/
1274 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1275 {
1276         struct i40e_mac_filter *f;
1277         u64 key;
1278
1279         if (!vsi || !macaddr)
1280                 return NULL;
1281
1282         key = i40e_addr_to_hkey(macaddr);
1283         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1284                 if ((ether_addr_equal(macaddr, f->macaddr)))
1285                         return f;
1286         }
1287         return NULL;
1288 }
1289
1290 /**
1291  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1292  * @vsi: the VSI to be searched
1293  *
1294  * Returns true if VSI is in vlan mode or false otherwise
1295  **/
1296 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1297 {
1298         /* If we have a PVID, always operate in VLAN mode */
1299         if (vsi->info.pvid)
1300                 return true;
1301
1302         /* We need to operate in VLAN mode whenever we have any filters with
1303          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1304          * time, incurring search cost repeatedly. However, we can notice two
1305          * things:
1306          *
1307          * 1) the only place where we can gain a VLAN filter is in
1308          *    i40e_add_filter.
1309          *
1310          * 2) the only place where filters are actually removed is in
1311          *    i40e_sync_filters_subtask.
1312          *
1313          * Thus, we can simply use a boolean value, has_vlan_filters which we
1314          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1315          * we have to perform the full search after deleting filters in
1316          * i40e_sync_filters_subtask, but we already have to search
1317          * filters here and can perform the check at the same time. This
1318          * results in avoiding embedding a loop for VLAN mode inside another
1319          * loop over all the filters, and should maintain correctness as noted
1320          * above.
1321          */
1322         return vsi->has_vlan_filter;
1323 }
1324
1325 /**
1326  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1327  * @vsi: the VSI to configure
1328  * @tmp_add_list: list of filters ready to be added
1329  * @tmp_del_list: list of filters ready to be deleted
1330  * @vlan_filters: the number of active VLAN filters
1331  *
1332  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1333  * behave as expected. If we have any active VLAN filters remaining or about
1334  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1335  * so that they only match against untagged traffic. If we no longer have any
1336  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1337  * so that they match against both tagged and untagged traffic. In this way,
1338  * we ensure that we correctly receive the desired traffic. This ensures that
1339  * when we have an active VLAN we will receive only untagged traffic and
1340  * traffic matching active VLANs. If we have no active VLANs then we will
1341  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1342  *
1343  * Finally, in a similar fashion, this function also corrects filters when
1344  * there is an active PVID assigned to this VSI.
1345  *
1346  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1347  *
1348  * This function is only expected to be called from within
1349  * i40e_sync_vsi_filters.
1350  *
1351  * NOTE: This function expects to be called while under the
1352  * mac_filter_hash_lock
1353  */
1354 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1355                                          struct hlist_head *tmp_add_list,
1356                                          struct hlist_head *tmp_del_list,
1357                                          int vlan_filters)
1358 {
1359         s16 pvid = le16_to_cpu(vsi->info.pvid);
1360         struct i40e_mac_filter *f, *add_head;
1361         struct i40e_new_mac_filter *new;
1362         struct hlist_node *h;
1363         int bkt, new_vlan;
1364
1365         /* To determine if a particular filter needs to be replaced we
1366          * have the three following conditions:
1367          *
1368          * a) if we have a PVID assigned, then all filters which are
1369          *    not marked as VLAN=PVID must be replaced with filters that
1370          *    are.
1371          * b) otherwise, if we have any active VLANS, all filters
1372          *    which are marked as VLAN=-1 must be replaced with
1373          *    filters marked as VLAN=0
1374          * c) finally, if we do not have any active VLANS, all filters
1375          *    which are marked as VLAN=0 must be replaced with filters
1376          *    marked as VLAN=-1
1377          */
1378
1379         /* Update the filters about to be added in place */
1380         hlist_for_each_entry(new, tmp_add_list, hlist) {
1381                 if (pvid && new->f->vlan != pvid)
1382                         new->f->vlan = pvid;
1383                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1384                         new->f->vlan = 0;
1385                 else if (!vlan_filters && new->f->vlan == 0)
1386                         new->f->vlan = I40E_VLAN_ANY;
1387         }
1388
1389         /* Update the remaining active filters */
1390         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1391                 /* Combine the checks for whether a filter needs to be changed
1392                  * and then determine the new VLAN inside the if block, in
1393                  * order to avoid duplicating code for adding the new filter
1394                  * then deleting the old filter.
1395                  */
1396                 if ((pvid && f->vlan != pvid) ||
1397                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1398                     (!vlan_filters && f->vlan == 0)) {
1399                         /* Determine the new vlan we will be adding */
1400                         if (pvid)
1401                                 new_vlan = pvid;
1402                         else if (vlan_filters)
1403                                 new_vlan = 0;
1404                         else
1405                                 new_vlan = I40E_VLAN_ANY;
1406
1407                         /* Create the new filter */
1408                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1409                         if (!add_head)
1410                                 return -ENOMEM;
1411
1412                         /* Create a temporary i40e_new_mac_filter */
1413                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1414                         if (!new)
1415                                 return -ENOMEM;
1416
1417                         new->f = add_head;
1418                         new->state = add_head->state;
1419
1420                         /* Add the new filter to the tmp list */
1421                         hlist_add_head(&new->hlist, tmp_add_list);
1422
1423                         /* Put the original filter into the delete list */
1424                         f->state = I40E_FILTER_REMOVE;
1425                         hash_del(&f->hlist);
1426                         hlist_add_head(&f->hlist, tmp_del_list);
1427                 }
1428         }
1429
1430         vsi->has_vlan_filter = !!vlan_filters;
1431
1432         return 0;
1433 }
1434
1435 /**
1436  * i40e_get_vf_new_vlan - Get new vlan id on a vf
1437  * @vsi: the vsi to configure
1438  * @new_mac: new mac filter to be added
1439  * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1440  * @vlan_filters: the number of active VLAN filters
1441  * @trusted: flag if the VF is trusted
1442  *
1443  * Get new VLAN id based on current VLAN filters, trust, PVID
1444  * and vf-vlan-prune-disable flag.
1445  *
1446  * Returns the value of the new vlan filter or
1447  * the old value if no new filter is needed.
1448  */
1449 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1450                                 struct i40e_new_mac_filter *new_mac,
1451                                 struct i40e_mac_filter *f,
1452                                 int vlan_filters,
1453                                 bool trusted)
1454 {
1455         s16 pvid = le16_to_cpu(vsi->info.pvid);
1456         struct i40e_pf *pf = vsi->back;
1457         bool is_any;
1458
1459         if (new_mac)
1460                 f = new_mac->f;
1461
1462         if (pvid && f->vlan != pvid)
1463                 return pvid;
1464
1465         is_any = (trusted ||
1466                   !(pf->flags & I40E_FLAG_VF_VLAN_PRUNING));
1467
1468         if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1469             (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1470             (is_any && !vlan_filters && f->vlan == 0)) {
1471                 if (is_any)
1472                         return I40E_VLAN_ANY;
1473                 else
1474                         return 0;
1475         }
1476
1477         return f->vlan;
1478 }
1479
1480 /**
1481  * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1482  * @vsi: the vsi to configure
1483  * @tmp_add_list: list of filters ready to be added
1484  * @tmp_del_list: list of filters ready to be deleted
1485  * @vlan_filters: the number of active VLAN filters
1486  * @trusted: flag if the VF is trusted
1487  *
1488  * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1489  * and vf-vlan-prune-disable flag.
1490  *
1491  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1492  *
1493  * This function is only expected to be called from within
1494  * i40e_sync_vsi_filters.
1495  *
1496  * NOTE: This function expects to be called while under the
1497  * mac_filter_hash_lock
1498  */
1499 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1500                                             struct hlist_head *tmp_add_list,
1501                                             struct hlist_head *tmp_del_list,
1502                                             int vlan_filters,
1503                                             bool trusted)
1504 {
1505         struct i40e_mac_filter *f, *add_head;
1506         struct i40e_new_mac_filter *new_mac;
1507         struct hlist_node *h;
1508         int bkt, new_vlan;
1509
1510         hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1511                 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1512                                                         vlan_filters, trusted);
1513         }
1514
1515         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1516                 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1517                                                 trusted);
1518                 if (new_vlan != f->vlan) {
1519                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1520                         if (!add_head)
1521                                 return -ENOMEM;
1522                         /* Create a temporary i40e_new_mac_filter */
1523                         new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1524                         if (!new_mac)
1525                                 return -ENOMEM;
1526                         new_mac->f = add_head;
1527                         new_mac->state = add_head->state;
1528
1529                         /* Add the new filter to the tmp list */
1530                         hlist_add_head(&new_mac->hlist, tmp_add_list);
1531
1532                         /* Put the original filter into the delete list */
1533                         f->state = I40E_FILTER_REMOVE;
1534                         hash_del(&f->hlist);
1535                         hlist_add_head(&f->hlist, tmp_del_list);
1536                 }
1537         }
1538
1539         vsi->has_vlan_filter = !!vlan_filters;
1540         return 0;
1541 }
1542
1543 /**
1544  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1545  * @vsi: the PF Main VSI - inappropriate for any other VSI
1546  * @macaddr: the MAC address
1547  *
1548  * Remove whatever filter the firmware set up so the driver can manage
1549  * its own filtering intelligently.
1550  **/
1551 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1552 {
1553         struct i40e_aqc_remove_macvlan_element_data element;
1554         struct i40e_pf *pf = vsi->back;
1555
1556         /* Only appropriate for the PF main VSI */
1557         if (vsi->type != I40E_VSI_MAIN)
1558                 return;
1559
1560         memset(&element, 0, sizeof(element));
1561         ether_addr_copy(element.mac_addr, macaddr);
1562         element.vlan_tag = 0;
1563         /* Ignore error returns, some firmware does it this way... */
1564         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1565         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1566
1567         memset(&element, 0, sizeof(element));
1568         ether_addr_copy(element.mac_addr, macaddr);
1569         element.vlan_tag = 0;
1570         /* ...and some firmware does it this way. */
1571         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1572                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1573         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1574 }
1575
1576 /**
1577  * i40e_add_filter - Add a mac/vlan filter to the VSI
1578  * @vsi: the VSI to be searched
1579  * @macaddr: the MAC address
1580  * @vlan: the vlan
1581  *
1582  * Returns ptr to the filter object or NULL when no memory available.
1583  *
1584  * NOTE: This function is expected to be called with mac_filter_hash_lock
1585  * being held.
1586  **/
1587 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1588                                         const u8 *macaddr, s16 vlan)
1589 {
1590         struct i40e_mac_filter *f;
1591         u64 key;
1592
1593         if (!vsi || !macaddr)
1594                 return NULL;
1595
1596         f = i40e_find_filter(vsi, macaddr, vlan);
1597         if (!f) {
1598                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1599                 if (!f)
1600                         return NULL;
1601
1602                 /* Update the boolean indicating if we need to function in
1603                  * VLAN mode.
1604                  */
1605                 if (vlan >= 0)
1606                         vsi->has_vlan_filter = true;
1607
1608                 ether_addr_copy(f->macaddr, macaddr);
1609                 f->vlan = vlan;
1610                 f->state = I40E_FILTER_NEW;
1611                 INIT_HLIST_NODE(&f->hlist);
1612
1613                 key = i40e_addr_to_hkey(macaddr);
1614                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1615
1616                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1617                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1618         }
1619
1620         /* If we're asked to add a filter that has been marked for removal, it
1621          * is safe to simply restore it to active state. __i40e_del_filter
1622          * will have simply deleted any filters which were previously marked
1623          * NEW or FAILED, so if it is currently marked REMOVE it must have
1624          * previously been ACTIVE. Since we haven't yet run the sync filters
1625          * task, just restore this filter to the ACTIVE state so that the
1626          * sync task leaves it in place
1627          */
1628         if (f->state == I40E_FILTER_REMOVE)
1629                 f->state = I40E_FILTER_ACTIVE;
1630
1631         return f;
1632 }
1633
1634 /**
1635  * __i40e_del_filter - Remove a specific filter from the VSI
1636  * @vsi: VSI to remove from
1637  * @f: the filter to remove from the list
1638  *
1639  * This function should be called instead of i40e_del_filter only if you know
1640  * the exact filter you will remove already, such as via i40e_find_filter or
1641  * i40e_find_mac.
1642  *
1643  * NOTE: This function is expected to be called with mac_filter_hash_lock
1644  * being held.
1645  * ANOTHER NOTE: This function MUST be called from within the context of
1646  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1647  * instead of list_for_each_entry().
1648  **/
1649 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1650 {
1651         if (!f)
1652                 return;
1653
1654         /* If the filter was never added to firmware then we can just delete it
1655          * directly and we don't want to set the status to remove or else an
1656          * admin queue command will unnecessarily fire.
1657          */
1658         if ((f->state == I40E_FILTER_FAILED) ||
1659             (f->state == I40E_FILTER_NEW)) {
1660                 hash_del(&f->hlist);
1661                 kfree(f);
1662         } else {
1663                 f->state = I40E_FILTER_REMOVE;
1664         }
1665
1666         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1667         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1668 }
1669
1670 /**
1671  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1672  * @vsi: the VSI to be searched
1673  * @macaddr: the MAC address
1674  * @vlan: the VLAN
1675  *
1676  * NOTE: This function is expected to be called with mac_filter_hash_lock
1677  * being held.
1678  * ANOTHER NOTE: This function MUST be called from within the context of
1679  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1680  * instead of list_for_each_entry().
1681  **/
1682 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1683 {
1684         struct i40e_mac_filter *f;
1685
1686         if (!vsi || !macaddr)
1687                 return;
1688
1689         f = i40e_find_filter(vsi, macaddr, vlan);
1690         __i40e_del_filter(vsi, f);
1691 }
1692
1693 /**
1694  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1695  * @vsi: the VSI to be searched
1696  * @macaddr: the mac address to be filtered
1697  *
1698  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1699  * go through all the macvlan filters and add a macvlan filter for each
1700  * unique vlan that already exists. If a PVID has been assigned, instead only
1701  * add the macaddr to that VLAN.
1702  *
1703  * Returns last filter added on success, else NULL
1704  **/
1705 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1706                                             const u8 *macaddr)
1707 {
1708         struct i40e_mac_filter *f, *add = NULL;
1709         struct hlist_node *h;
1710         int bkt;
1711
1712         if (vsi->info.pvid)
1713                 return i40e_add_filter(vsi, macaddr,
1714                                        le16_to_cpu(vsi->info.pvid));
1715
1716         if (!i40e_is_vsi_in_vlan(vsi))
1717                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1718
1719         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1720                 if (f->state == I40E_FILTER_REMOVE)
1721                         continue;
1722                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1723                 if (!add)
1724                         return NULL;
1725         }
1726
1727         return add;
1728 }
1729
1730 /**
1731  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1732  * @vsi: the VSI to be searched
1733  * @macaddr: the mac address to be removed
1734  *
1735  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1736  * associated with.
1737  *
1738  * Returns 0 for success, or error
1739  **/
1740 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1741 {
1742         struct i40e_mac_filter *f;
1743         struct hlist_node *h;
1744         bool found = false;
1745         int bkt;
1746
1747         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1748         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1749                 if (ether_addr_equal(macaddr, f->macaddr)) {
1750                         __i40e_del_filter(vsi, f);
1751                         found = true;
1752                 }
1753         }
1754
1755         if (found)
1756                 return 0;
1757         else
1758                 return -ENOENT;
1759 }
1760
1761 /**
1762  * i40e_set_mac - NDO callback to set mac address
1763  * @netdev: network interface device structure
1764  * @p: pointer to an address structure
1765  *
1766  * Returns 0 on success, negative on failure
1767  **/
1768 static int i40e_set_mac(struct net_device *netdev, void *p)
1769 {
1770         struct i40e_netdev_priv *np = netdev_priv(netdev);
1771         struct i40e_vsi *vsi = np->vsi;
1772         struct i40e_pf *pf = vsi->back;
1773         struct i40e_hw *hw = &pf->hw;
1774         struct sockaddr *addr = p;
1775
1776         if (!is_valid_ether_addr(addr->sa_data))
1777                 return -EADDRNOTAVAIL;
1778
1779         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1780                 netdev_info(netdev, "already using mac address %pM\n",
1781                             addr->sa_data);
1782                 return 0;
1783         }
1784
1785         if (test_bit(__I40E_DOWN, pf->state) ||
1786             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1787                 return -EADDRNOTAVAIL;
1788
1789         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1790                 netdev_info(netdev, "returning to hw mac address %pM\n",
1791                             hw->mac.addr);
1792         else
1793                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1794
1795         /* Copy the address first, so that we avoid a possible race with
1796          * .set_rx_mode().
1797          * - Remove old address from MAC filter
1798          * - Copy new address
1799          * - Add new address to MAC filter
1800          */
1801         spin_lock_bh(&vsi->mac_filter_hash_lock);
1802         i40e_del_mac_filter(vsi, netdev->dev_addr);
1803         eth_hw_addr_set(netdev, addr->sa_data);
1804         i40e_add_mac_filter(vsi, netdev->dev_addr);
1805         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1806
1807         if (vsi->type == I40E_VSI_MAIN) {
1808                 int ret;
1809
1810                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1811                                                 addr->sa_data, NULL);
1812                 if (ret)
1813                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %d, AQ ret %s\n",
1814                                     ret,
1815                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1816         }
1817
1818         /* schedule our worker thread which will take care of
1819          * applying the new filter changes
1820          */
1821         i40e_service_event_schedule(pf);
1822         return 0;
1823 }
1824
1825 /**
1826  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1827  * @vsi: vsi structure
1828  * @seed: RSS hash seed
1829  * @lut: pointer to lookup table of lut_size
1830  * @lut_size: size of the lookup table
1831  **/
1832 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1833                               u8 *lut, u16 lut_size)
1834 {
1835         struct i40e_pf *pf = vsi->back;
1836         struct i40e_hw *hw = &pf->hw;
1837         int ret = 0;
1838
1839         if (seed) {
1840                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1841                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1842                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1843                 if (ret) {
1844                         dev_info(&pf->pdev->dev,
1845                                  "Cannot set RSS key, err %d aq_err %s\n",
1846                                  ret,
1847                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1848                         return ret;
1849                 }
1850         }
1851         if (lut) {
1852                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1853
1854                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1855                 if (ret) {
1856                         dev_info(&pf->pdev->dev,
1857                                  "Cannot set RSS lut, err %d aq_err %s\n",
1858                                  ret,
1859                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1860                         return ret;
1861                 }
1862         }
1863         return ret;
1864 }
1865
1866 /**
1867  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1868  * @vsi: VSI structure
1869  **/
1870 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1871 {
1872         struct i40e_pf *pf = vsi->back;
1873         u8 seed[I40E_HKEY_ARRAY_SIZE];
1874         u8 *lut;
1875         int ret;
1876
1877         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1878                 return 0;
1879         if (!vsi->rss_size)
1880                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1881                                       vsi->num_queue_pairs);
1882         if (!vsi->rss_size)
1883                 return -EINVAL;
1884         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1885         if (!lut)
1886                 return -ENOMEM;
1887
1888         /* Use the user configured hash keys and lookup table if there is one,
1889          * otherwise use default
1890          */
1891         if (vsi->rss_lut_user)
1892                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1893         else
1894                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1895         if (vsi->rss_hkey_user)
1896                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1897         else
1898                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1899         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1900         kfree(lut);
1901         return ret;
1902 }
1903
1904 /**
1905  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1906  * @vsi: the VSI being configured,
1907  * @ctxt: VSI context structure
1908  * @enabled_tc: number of traffic classes to enable
1909  *
1910  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1911  **/
1912 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1913                                            struct i40e_vsi_context *ctxt,
1914                                            u8 enabled_tc)
1915 {
1916         u16 qcount = 0, max_qcount, qmap, sections = 0;
1917         int i, override_q, pow, num_qps, ret;
1918         u8 netdev_tc = 0, offset = 0;
1919
1920         if (vsi->type != I40E_VSI_MAIN)
1921                 return -EINVAL;
1922         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1923         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1924         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1925         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1926         num_qps = vsi->mqprio_qopt.qopt.count[0];
1927
1928         /* find the next higher power-of-2 of num queue pairs */
1929         pow = ilog2(num_qps);
1930         if (!is_power_of_2(num_qps))
1931                 pow++;
1932         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1933                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1934
1935         /* Setup queue offset/count for all TCs for given VSI */
1936         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1937         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1938                 /* See if the given TC is enabled for the given VSI */
1939                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1940                         offset = vsi->mqprio_qopt.qopt.offset[i];
1941                         qcount = vsi->mqprio_qopt.qopt.count[i];
1942                         if (qcount > max_qcount)
1943                                 max_qcount = qcount;
1944                         vsi->tc_config.tc_info[i].qoffset = offset;
1945                         vsi->tc_config.tc_info[i].qcount = qcount;
1946                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1947                 } else {
1948                         /* TC is not enabled so set the offset to
1949                          * default queue and allocate one queue
1950                          * for the given TC.
1951                          */
1952                         vsi->tc_config.tc_info[i].qoffset = 0;
1953                         vsi->tc_config.tc_info[i].qcount = 1;
1954                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1955                 }
1956         }
1957
1958         /* Set actual Tx/Rx queue pairs */
1959         vsi->num_queue_pairs = offset + qcount;
1960
1961         /* Setup queue TC[0].qmap for given VSI context */
1962         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1963         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1964         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1965         ctxt->info.valid_sections |= cpu_to_le16(sections);
1966
1967         /* Reconfigure RSS for main VSI with max queue count */
1968         vsi->rss_size = max_qcount;
1969         ret = i40e_vsi_config_rss(vsi);
1970         if (ret) {
1971                 dev_info(&vsi->back->pdev->dev,
1972                          "Failed to reconfig rss for num_queues (%u)\n",
1973                          max_qcount);
1974                 return ret;
1975         }
1976         vsi->reconfig_rss = true;
1977         dev_dbg(&vsi->back->pdev->dev,
1978                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1979
1980         /* Find queue count available for channel VSIs and starting offset
1981          * for channel VSIs
1982          */
1983         override_q = vsi->mqprio_qopt.qopt.count[0];
1984         if (override_q && override_q < vsi->num_queue_pairs) {
1985                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1986                 vsi->next_base_queue = override_q;
1987         }
1988         return 0;
1989 }
1990
1991 /**
1992  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1993  * @vsi: the VSI being setup
1994  * @ctxt: VSI context structure
1995  * @enabled_tc: Enabled TCs bitmap
1996  * @is_add: True if called before Add VSI
1997  *
1998  * Setup VSI queue mapping for enabled traffic classes.
1999  **/
2000 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2001                                      struct i40e_vsi_context *ctxt,
2002                                      u8 enabled_tc,
2003                                      bool is_add)
2004 {
2005         struct i40e_pf *pf = vsi->back;
2006         u16 num_tc_qps = 0;
2007         u16 sections = 0;
2008         u8 netdev_tc = 0;
2009         u16 numtc = 1;
2010         u16 qcount;
2011         u8 offset;
2012         u16 qmap;
2013         int i;
2014
2015         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2016         offset = 0;
2017         /* zero out queue mapping, it will get updated on the end of the function */
2018         memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2019
2020         if (vsi->type == I40E_VSI_MAIN) {
2021                 /* This code helps add more queue to the VSI if we have
2022                  * more cores than RSS can support, the higher cores will
2023                  * be served by ATR or other filters. Furthermore, the
2024                  * non-zero req_queue_pairs says that user requested a new
2025                  * queue count via ethtool's set_channels, so use this
2026                  * value for queues distribution across traffic classes
2027                  * We need at least one queue pair for the interface
2028                  * to be usable as we see in else statement.
2029                  */
2030                 if (vsi->req_queue_pairs > 0)
2031                         vsi->num_queue_pairs = vsi->req_queue_pairs;
2032                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2033                         vsi->num_queue_pairs = pf->num_lan_msix;
2034                 else
2035                         vsi->num_queue_pairs = 1;
2036         }
2037
2038         /* Number of queues per enabled TC */
2039         if (vsi->type == I40E_VSI_MAIN ||
2040             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2041                 num_tc_qps = vsi->num_queue_pairs;
2042         else
2043                 num_tc_qps = vsi->alloc_queue_pairs;
2044
2045         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
2046                 /* Find numtc from enabled TC bitmap */
2047                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2048                         if (enabled_tc & BIT(i)) /* TC is enabled */
2049                                 numtc++;
2050                 }
2051                 if (!numtc) {
2052                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2053                         numtc = 1;
2054                 }
2055                 num_tc_qps = num_tc_qps / numtc;
2056                 num_tc_qps = min_t(int, num_tc_qps,
2057                                    i40e_pf_get_max_q_per_tc(pf));
2058         }
2059
2060         vsi->tc_config.numtc = numtc;
2061         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2062
2063         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
2064         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2065                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2066
2067         /* Setup queue offset/count for all TCs for given VSI */
2068         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2069                 /* See if the given TC is enabled for the given VSI */
2070                 if (vsi->tc_config.enabled_tc & BIT(i)) {
2071                         /* TC is enabled */
2072                         int pow, num_qps;
2073
2074                         switch (vsi->type) {
2075                         case I40E_VSI_MAIN:
2076                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
2077                                     I40E_FLAG_FD_ATR_ENABLED)) ||
2078                                     vsi->tc_config.enabled_tc != 1) {
2079                                         qcount = min_t(int, pf->alloc_rss_size,
2080                                                        num_tc_qps);
2081                                         break;
2082                                 }
2083                                 fallthrough;
2084                         case I40E_VSI_FDIR:
2085                         case I40E_VSI_SRIOV:
2086                         case I40E_VSI_VMDQ2:
2087                         default:
2088                                 qcount = num_tc_qps;
2089                                 WARN_ON(i != 0);
2090                                 break;
2091                         }
2092                         vsi->tc_config.tc_info[i].qoffset = offset;
2093                         vsi->tc_config.tc_info[i].qcount = qcount;
2094
2095                         /* find the next higher power-of-2 of num queue pairs */
2096                         num_qps = qcount;
2097                         pow = 0;
2098                         while (num_qps && (BIT_ULL(pow) < qcount)) {
2099                                 pow++;
2100                                 num_qps >>= 1;
2101                         }
2102
2103                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2104                         qmap =
2105                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2106                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2107
2108                         offset += qcount;
2109                 } else {
2110                         /* TC is not enabled so set the offset to
2111                          * default queue and allocate one queue
2112                          * for the given TC.
2113                          */
2114                         vsi->tc_config.tc_info[i].qoffset = 0;
2115                         vsi->tc_config.tc_info[i].qcount = 1;
2116                         vsi->tc_config.tc_info[i].netdev_tc = 0;
2117
2118                         qmap = 0;
2119                 }
2120                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2121         }
2122         /* Do not change previously set num_queue_pairs for PFs and VFs*/
2123         if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2124             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2125             (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2126                 vsi->num_queue_pairs = offset;
2127
2128         /* Scheduler section valid can only be set for ADD VSI */
2129         if (is_add) {
2130                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2131
2132                 ctxt->info.up_enable_bits = enabled_tc;
2133         }
2134         if (vsi->type == I40E_VSI_SRIOV) {
2135                 ctxt->info.mapping_flags |=
2136                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2137                 for (i = 0; i < vsi->num_queue_pairs; i++)
2138                         ctxt->info.queue_mapping[i] =
2139                                                cpu_to_le16(vsi->base_queue + i);
2140         } else {
2141                 ctxt->info.mapping_flags |=
2142                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2143                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2144         }
2145         ctxt->info.valid_sections |= cpu_to_le16(sections);
2146 }
2147
2148 /**
2149  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2150  * @netdev: the netdevice
2151  * @addr: address to add
2152  *
2153  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2154  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2155  */
2156 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2157 {
2158         struct i40e_netdev_priv *np = netdev_priv(netdev);
2159         struct i40e_vsi *vsi = np->vsi;
2160
2161         if (i40e_add_mac_filter(vsi, addr))
2162                 return 0;
2163         else
2164                 return -ENOMEM;
2165 }
2166
2167 /**
2168  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2169  * @netdev: the netdevice
2170  * @addr: address to add
2171  *
2172  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2173  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2174  */
2175 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2176 {
2177         struct i40e_netdev_priv *np = netdev_priv(netdev);
2178         struct i40e_vsi *vsi = np->vsi;
2179
2180         /* Under some circumstances, we might receive a request to delete
2181          * our own device address from our uc list. Because we store the
2182          * device address in the VSI's MAC/VLAN filter list, we need to ignore
2183          * such requests and not delete our device address from this list.
2184          */
2185         if (ether_addr_equal(addr, netdev->dev_addr))
2186                 return 0;
2187
2188         i40e_del_mac_filter(vsi, addr);
2189
2190         return 0;
2191 }
2192
2193 /**
2194  * i40e_set_rx_mode - NDO callback to set the netdev filters
2195  * @netdev: network interface device structure
2196  **/
2197 static void i40e_set_rx_mode(struct net_device *netdev)
2198 {
2199         struct i40e_netdev_priv *np = netdev_priv(netdev);
2200         struct i40e_vsi *vsi = np->vsi;
2201
2202         spin_lock_bh(&vsi->mac_filter_hash_lock);
2203
2204         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2205         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2206
2207         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2208
2209         /* check for other flag changes */
2210         if (vsi->current_netdev_flags != vsi->netdev->flags) {
2211                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2212                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2213         }
2214 }
2215
2216 /**
2217  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2218  * @vsi: Pointer to VSI struct
2219  * @from: Pointer to list which contains MAC filter entries - changes to
2220  *        those entries needs to be undone.
2221  *
2222  * MAC filter entries from this list were slated for deletion.
2223  **/
2224 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2225                                          struct hlist_head *from)
2226 {
2227         struct i40e_mac_filter *f;
2228         struct hlist_node *h;
2229
2230         hlist_for_each_entry_safe(f, h, from, hlist) {
2231                 u64 key = i40e_addr_to_hkey(f->macaddr);
2232
2233                 /* Move the element back into MAC filter list*/
2234                 hlist_del(&f->hlist);
2235                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2236         }
2237 }
2238
2239 /**
2240  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2241  * @vsi: Pointer to vsi struct
2242  * @from: Pointer to list which contains MAC filter entries - changes to
2243  *        those entries needs to be undone.
2244  *
2245  * MAC filter entries from this list were slated for addition.
2246  **/
2247 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2248                                          struct hlist_head *from)
2249 {
2250         struct i40e_new_mac_filter *new;
2251         struct hlist_node *h;
2252
2253         hlist_for_each_entry_safe(new, h, from, hlist) {
2254                 /* We can simply free the wrapper structure */
2255                 hlist_del(&new->hlist);
2256                 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2257                 kfree(new);
2258         }
2259 }
2260
2261 /**
2262  * i40e_next_filter - Get the next non-broadcast filter from a list
2263  * @next: pointer to filter in list
2264  *
2265  * Returns the next non-broadcast filter in the list. Required so that we
2266  * ignore broadcast filters within the list, since these are not handled via
2267  * the normal firmware update path.
2268  */
2269 static
2270 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2271 {
2272         hlist_for_each_entry_continue(next, hlist) {
2273                 if (!is_broadcast_ether_addr(next->f->macaddr))
2274                         return next;
2275         }
2276
2277         return NULL;
2278 }
2279
2280 /**
2281  * i40e_update_filter_state - Update filter state based on return data
2282  * from firmware
2283  * @count: Number of filters added
2284  * @add_list: return data from fw
2285  * @add_head: pointer to first filter in current batch
2286  *
2287  * MAC filter entries from list were slated to be added to device. Returns
2288  * number of successful filters. Note that 0 does NOT mean success!
2289  **/
2290 static int
2291 i40e_update_filter_state(int count,
2292                          struct i40e_aqc_add_macvlan_element_data *add_list,
2293                          struct i40e_new_mac_filter *add_head)
2294 {
2295         int retval = 0;
2296         int i;
2297
2298         for (i = 0; i < count; i++) {
2299                 /* Always check status of each filter. We don't need to check
2300                  * the firmware return status because we pre-set the filter
2301                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2302                  * request to the adminq. Thus, if it no longer matches then
2303                  * we know the filter is active.
2304                  */
2305                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2306                         add_head->state = I40E_FILTER_FAILED;
2307                 } else {
2308                         add_head->state = I40E_FILTER_ACTIVE;
2309                         retval++;
2310                 }
2311
2312                 add_head = i40e_next_filter(add_head);
2313                 if (!add_head)
2314                         break;
2315         }
2316
2317         return retval;
2318 }
2319
2320 /**
2321  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2322  * @vsi: ptr to the VSI
2323  * @vsi_name: name to display in messages
2324  * @list: the list of filters to send to firmware
2325  * @num_del: the number of filters to delete
2326  * @retval: Set to -EIO on failure to delete
2327  *
2328  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2329  * *retval instead of a return value so that success does not force ret_val to
2330  * be set to 0. This ensures that a sequence of calls to this function
2331  * preserve the previous value of *retval on successful delete.
2332  */
2333 static
2334 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2335                           struct i40e_aqc_remove_macvlan_element_data *list,
2336                           int num_del, int *retval)
2337 {
2338         struct i40e_hw *hw = &vsi->back->hw;
2339         int aq_ret;
2340         int aq_err;
2341
2342         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2343         aq_err = hw->aq.asq_last_status;
2344
2345         /* Explicitly ignore and do not report when firmware returns ENOENT */
2346         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2347                 *retval = -EIO;
2348                 dev_info(&vsi->back->pdev->dev,
2349                          "ignoring delete macvlan error on %s, err %d, aq_err %s\n",
2350                          vsi_name, aq_ret,
2351                          i40e_aq_str(hw, aq_err));
2352         }
2353 }
2354
2355 /**
2356  * i40e_aqc_add_filters - Request firmware to add a set of filters
2357  * @vsi: ptr to the VSI
2358  * @vsi_name: name to display in messages
2359  * @list: the list of filters to send to firmware
2360  * @add_head: Position in the add hlist
2361  * @num_add: the number of filters to add
2362  *
2363  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2364  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2365  * space for more filters.
2366  */
2367 static
2368 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2369                           struct i40e_aqc_add_macvlan_element_data *list,
2370                           struct i40e_new_mac_filter *add_head,
2371                           int num_add)
2372 {
2373         struct i40e_hw *hw = &vsi->back->hw;
2374         int aq_err, fcnt;
2375
2376         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2377         aq_err = hw->aq.asq_last_status;
2378         fcnt = i40e_update_filter_state(num_add, list, add_head);
2379
2380         if (fcnt != num_add) {
2381                 if (vsi->type == I40E_VSI_MAIN) {
2382                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2383                         dev_warn(&vsi->back->pdev->dev,
2384                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2385                                  i40e_aq_str(hw, aq_err), vsi_name);
2386                 } else if (vsi->type == I40E_VSI_SRIOV ||
2387                            vsi->type == I40E_VSI_VMDQ1 ||
2388                            vsi->type == I40E_VSI_VMDQ2) {
2389                         dev_warn(&vsi->back->pdev->dev,
2390                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2391                                  i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2392                 } else {
2393                         dev_warn(&vsi->back->pdev->dev,
2394                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2395                                  i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2396                 }
2397         }
2398 }
2399
2400 /**
2401  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2402  * @vsi: pointer to the VSI
2403  * @vsi_name: the VSI name
2404  * @f: filter data
2405  *
2406  * This function sets or clears the promiscuous broadcast flags for VLAN
2407  * filters in order to properly receive broadcast frames. Assumes that only
2408  * broadcast filters are passed.
2409  *
2410  * Returns status indicating success or failure;
2411  **/
2412 static int
2413 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2414                           struct i40e_mac_filter *f)
2415 {
2416         bool enable = f->state == I40E_FILTER_NEW;
2417         struct i40e_hw *hw = &vsi->back->hw;
2418         int aq_ret;
2419
2420         if (f->vlan == I40E_VLAN_ANY) {
2421                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2422                                                    vsi->seid,
2423                                                    enable,
2424                                                    NULL);
2425         } else {
2426                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2427                                                             vsi->seid,
2428                                                             enable,
2429                                                             f->vlan,
2430                                                             NULL);
2431         }
2432
2433         if (aq_ret) {
2434                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2435                 dev_warn(&vsi->back->pdev->dev,
2436                          "Error %s, forcing overflow promiscuous on %s\n",
2437                          i40e_aq_str(hw, hw->aq.asq_last_status),
2438                          vsi_name);
2439         }
2440
2441         return aq_ret;
2442 }
2443
2444 /**
2445  * i40e_set_promiscuous - set promiscuous mode
2446  * @pf: board private structure
2447  * @promisc: promisc on or off
2448  *
2449  * There are different ways of setting promiscuous mode on a PF depending on
2450  * what state/environment we're in.  This identifies and sets it appropriately.
2451  * Returns 0 on success.
2452  **/
2453 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2454 {
2455         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2456         struct i40e_hw *hw = &pf->hw;
2457         int aq_ret;
2458
2459         if (vsi->type == I40E_VSI_MAIN &&
2460             pf->lan_veb != I40E_NO_VEB &&
2461             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2462                 /* set defport ON for Main VSI instead of true promisc
2463                  * this way we will get all unicast/multicast and VLAN
2464                  * promisc behavior but will not get VF or VMDq traffic
2465                  * replicated on the Main VSI.
2466                  */
2467                 if (promisc)
2468                         aq_ret = i40e_aq_set_default_vsi(hw,
2469                                                          vsi->seid,
2470                                                          NULL);
2471                 else
2472                         aq_ret = i40e_aq_clear_default_vsi(hw,
2473                                                            vsi->seid,
2474                                                            NULL);
2475                 if (aq_ret) {
2476                         dev_info(&pf->pdev->dev,
2477                                  "Set default VSI failed, err %d, aq_err %s\n",
2478                                  aq_ret,
2479                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2480                 }
2481         } else {
2482                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2483                                                   hw,
2484                                                   vsi->seid,
2485                                                   promisc, NULL,
2486                                                   true);
2487                 if (aq_ret) {
2488                         dev_info(&pf->pdev->dev,
2489                                  "set unicast promisc failed, err %d, aq_err %s\n",
2490                                  aq_ret,
2491                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2492                 }
2493                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2494                                                   hw,
2495                                                   vsi->seid,
2496                                                   promisc, NULL);
2497                 if (aq_ret) {
2498                         dev_info(&pf->pdev->dev,
2499                                  "set multicast promisc failed, err %d, aq_err %s\n",
2500                                  aq_ret,
2501                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2502                 }
2503         }
2504
2505         if (!aq_ret)
2506                 pf->cur_promisc = promisc;
2507
2508         return aq_ret;
2509 }
2510
2511 /**
2512  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2513  * @vsi: ptr to the VSI
2514  *
2515  * Push any outstanding VSI filter changes through the AdminQ.
2516  *
2517  * Returns 0 or error value
2518  **/
2519 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2520 {
2521         struct hlist_head tmp_add_list, tmp_del_list;
2522         struct i40e_mac_filter *f;
2523         struct i40e_new_mac_filter *new, *add_head = NULL;
2524         struct i40e_hw *hw = &vsi->back->hw;
2525         bool old_overflow, new_overflow;
2526         unsigned int failed_filters = 0;
2527         unsigned int vlan_filters = 0;
2528         char vsi_name[16] = "PF";
2529         int filter_list_len = 0;
2530         u32 changed_flags = 0;
2531         struct hlist_node *h;
2532         struct i40e_pf *pf;
2533         int num_add = 0;
2534         int num_del = 0;
2535         int aq_ret = 0;
2536         int retval = 0;
2537         u16 cmd_flags;
2538         int list_size;
2539         int bkt;
2540
2541         /* empty array typed pointers, kcalloc later */
2542         struct i40e_aqc_add_macvlan_element_data *add_list;
2543         struct i40e_aqc_remove_macvlan_element_data *del_list;
2544
2545         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2546                 usleep_range(1000, 2000);
2547         pf = vsi->back;
2548
2549         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2550
2551         if (vsi->netdev) {
2552                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2553                 vsi->current_netdev_flags = vsi->netdev->flags;
2554         }
2555
2556         INIT_HLIST_HEAD(&tmp_add_list);
2557         INIT_HLIST_HEAD(&tmp_del_list);
2558
2559         if (vsi->type == I40E_VSI_SRIOV)
2560                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2561         else if (vsi->type != I40E_VSI_MAIN)
2562                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2563
2564         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2565                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2566
2567                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2568                 /* Create a list of filters to delete. */
2569                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2570                         if (f->state == I40E_FILTER_REMOVE) {
2571                                 /* Move the element into temporary del_list */
2572                                 hash_del(&f->hlist);
2573                                 hlist_add_head(&f->hlist, &tmp_del_list);
2574
2575                                 /* Avoid counting removed filters */
2576                                 continue;
2577                         }
2578                         if (f->state == I40E_FILTER_NEW) {
2579                                 /* Create a temporary i40e_new_mac_filter */
2580                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2581                                 if (!new)
2582                                         goto err_no_memory_locked;
2583
2584                                 /* Store pointer to the real filter */
2585                                 new->f = f;
2586                                 new->state = f->state;
2587
2588                                 /* Add it to the hash list */
2589                                 hlist_add_head(&new->hlist, &tmp_add_list);
2590                         }
2591
2592                         /* Count the number of active (current and new) VLAN
2593                          * filters we have now. Does not count filters which
2594                          * are marked for deletion.
2595                          */
2596                         if (f->vlan > 0)
2597                                 vlan_filters++;
2598                 }
2599
2600                 if (vsi->type != I40E_VSI_SRIOV)
2601                         retval = i40e_correct_mac_vlan_filters
2602                                 (vsi, &tmp_add_list, &tmp_del_list,
2603                                  vlan_filters);
2604                 else if (pf->vf)
2605                         retval = i40e_correct_vf_mac_vlan_filters
2606                                 (vsi, &tmp_add_list, &tmp_del_list,
2607                                  vlan_filters, pf->vf[vsi->vf_id].trusted);
2608
2609                 hlist_for_each_entry(new, &tmp_add_list, hlist)
2610                         netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2611
2612                 if (retval)
2613                         goto err_no_memory_locked;
2614
2615                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2616         }
2617
2618         /* Now process 'del_list' outside the lock */
2619         if (!hlist_empty(&tmp_del_list)) {
2620                 filter_list_len = hw->aq.asq_buf_size /
2621                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2622                 list_size = filter_list_len *
2623                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2624                 del_list = kzalloc(list_size, GFP_ATOMIC);
2625                 if (!del_list)
2626                         goto err_no_memory;
2627
2628                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2629                         cmd_flags = 0;
2630
2631                         /* handle broadcast filters by updating the broadcast
2632                          * promiscuous flag and release filter list.
2633                          */
2634                         if (is_broadcast_ether_addr(f->macaddr)) {
2635                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2636
2637                                 hlist_del(&f->hlist);
2638                                 kfree(f);
2639                                 continue;
2640                         }
2641
2642                         /* add to delete list */
2643                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2644                         if (f->vlan == I40E_VLAN_ANY) {
2645                                 del_list[num_del].vlan_tag = 0;
2646                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2647                         } else {
2648                                 del_list[num_del].vlan_tag =
2649                                         cpu_to_le16((u16)(f->vlan));
2650                         }
2651
2652                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2653                         del_list[num_del].flags = cmd_flags;
2654                         num_del++;
2655
2656                         /* flush a full buffer */
2657                         if (num_del == filter_list_len) {
2658                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2659                                                      num_del, &retval);
2660                                 memset(del_list, 0, list_size);
2661                                 num_del = 0;
2662                         }
2663                         /* Release memory for MAC filter entries which were
2664                          * synced up with HW.
2665                          */
2666                         hlist_del(&f->hlist);
2667                         kfree(f);
2668                 }
2669
2670                 if (num_del) {
2671                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2672                                              num_del, &retval);
2673                 }
2674
2675                 kfree(del_list);
2676                 del_list = NULL;
2677         }
2678
2679         if (!hlist_empty(&tmp_add_list)) {
2680                 /* Do all the adds now. */
2681                 filter_list_len = hw->aq.asq_buf_size /
2682                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2683                 list_size = filter_list_len *
2684                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2685                 add_list = kzalloc(list_size, GFP_ATOMIC);
2686                 if (!add_list)
2687                         goto err_no_memory;
2688
2689                 num_add = 0;
2690                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2691                         /* handle broadcast filters by updating the broadcast
2692                          * promiscuous flag instead of adding a MAC filter.
2693                          */
2694                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2695                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2696                                                               new->f))
2697                                         new->state = I40E_FILTER_FAILED;
2698                                 else
2699                                         new->state = I40E_FILTER_ACTIVE;
2700                                 continue;
2701                         }
2702
2703                         /* add to add array */
2704                         if (num_add == 0)
2705                                 add_head = new;
2706                         cmd_flags = 0;
2707                         ether_addr_copy(add_list[num_add].mac_addr,
2708                                         new->f->macaddr);
2709                         if (new->f->vlan == I40E_VLAN_ANY) {
2710                                 add_list[num_add].vlan_tag = 0;
2711                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2712                         } else {
2713                                 add_list[num_add].vlan_tag =
2714                                         cpu_to_le16((u16)(new->f->vlan));
2715                         }
2716                         add_list[num_add].queue_number = 0;
2717                         /* set invalid match method for later detection */
2718                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2719                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2720                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2721                         num_add++;
2722
2723                         /* flush a full buffer */
2724                         if (num_add == filter_list_len) {
2725                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2726                                                      add_head, num_add);
2727                                 memset(add_list, 0, list_size);
2728                                 num_add = 0;
2729                         }
2730                 }
2731                 if (num_add) {
2732                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2733                                              num_add);
2734                 }
2735                 /* Now move all of the filters from the temp add list back to
2736                  * the VSI's list.
2737                  */
2738                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2739                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2740                         /* Only update the state if we're still NEW */
2741                         if (new->f->state == I40E_FILTER_NEW)
2742                                 new->f->state = new->state;
2743                         hlist_del(&new->hlist);
2744                         netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2745                         kfree(new);
2746                 }
2747                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2748                 kfree(add_list);
2749                 add_list = NULL;
2750         }
2751
2752         /* Determine the number of active and failed filters. */
2753         spin_lock_bh(&vsi->mac_filter_hash_lock);
2754         vsi->active_filters = 0;
2755         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2756                 if (f->state == I40E_FILTER_ACTIVE)
2757                         vsi->active_filters++;
2758                 else if (f->state == I40E_FILTER_FAILED)
2759                         failed_filters++;
2760         }
2761         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2762
2763         /* Check if we are able to exit overflow promiscuous mode. We can
2764          * safely exit if we didn't just enter, we no longer have any failed
2765          * filters, and we have reduced filters below the threshold value.
2766          */
2767         if (old_overflow && !failed_filters &&
2768             vsi->active_filters < vsi->promisc_threshold) {
2769                 dev_info(&pf->pdev->dev,
2770                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2771                          vsi_name);
2772                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2773                 vsi->promisc_threshold = 0;
2774         }
2775
2776         /* if the VF is not trusted do not do promisc */
2777         if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2778             !pf->vf[vsi->vf_id].trusted) {
2779                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2780                 goto out;
2781         }
2782
2783         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2784
2785         /* If we are entering overflow promiscuous, we need to calculate a new
2786          * threshold for when we are safe to exit
2787          */
2788         if (!old_overflow && new_overflow)
2789                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2790
2791         /* check for changes in promiscuous modes */
2792         if (changed_flags & IFF_ALLMULTI) {
2793                 bool cur_multipromisc;
2794
2795                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2796                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2797                                                                vsi->seid,
2798                                                                cur_multipromisc,
2799                                                                NULL);
2800                 if (aq_ret) {
2801                         retval = i40e_aq_rc_to_posix(aq_ret,
2802                                                      hw->aq.asq_last_status);
2803                         dev_info(&pf->pdev->dev,
2804                                  "set multi promisc failed on %s, err %d aq_err %s\n",
2805                                  vsi_name,
2806                                  aq_ret,
2807                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2808                 } else {
2809                         dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2810                                  cur_multipromisc ? "entering" : "leaving");
2811                 }
2812         }
2813
2814         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2815                 bool cur_promisc;
2816
2817                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2818                                new_overflow);
2819                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2820                 if (aq_ret) {
2821                         retval = i40e_aq_rc_to_posix(aq_ret,
2822                                                      hw->aq.asq_last_status);
2823                         dev_info(&pf->pdev->dev,
2824                                  "Setting promiscuous %s failed on %s, err %d aq_err %s\n",
2825                                  cur_promisc ? "on" : "off",
2826                                  vsi_name,
2827                                  aq_ret,
2828                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2829                 }
2830         }
2831 out:
2832         /* if something went wrong then set the changed flag so we try again */
2833         if (retval)
2834                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2835
2836         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2837         return retval;
2838
2839 err_no_memory:
2840         /* Restore elements on the temporary add and delete lists */
2841         spin_lock_bh(&vsi->mac_filter_hash_lock);
2842 err_no_memory_locked:
2843         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2844         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2845         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2846
2847         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2848         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2849         return -ENOMEM;
2850 }
2851
2852 /**
2853  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2854  * @pf: board private structure
2855  **/
2856 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2857 {
2858         int v;
2859
2860         if (!pf)
2861                 return;
2862         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2863                 return;
2864         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2865                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2866                 return;
2867         }
2868
2869         for (v = 0; v < pf->num_alloc_vsi; v++) {
2870                 if (pf->vsi[v] &&
2871                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2872                     !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2873                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2874
2875                         if (ret) {
2876                                 /* come back and try again later */
2877                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2878                                         pf->state);
2879                                 break;
2880                         }
2881                 }
2882         }
2883 }
2884
2885 /**
2886  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2887  * @vsi: the vsi
2888  **/
2889 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2890 {
2891         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2892                 return I40E_RXBUFFER_2048;
2893         else
2894                 return I40E_RXBUFFER_3072;
2895 }
2896
2897 /**
2898  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2899  * @netdev: network interface device structure
2900  * @new_mtu: new value for maximum frame size
2901  *
2902  * Returns 0 on success, negative on failure
2903  **/
2904 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2905 {
2906         struct i40e_netdev_priv *np = netdev_priv(netdev);
2907         struct i40e_vsi *vsi = np->vsi;
2908         struct i40e_pf *pf = vsi->back;
2909
2910         if (i40e_enabled_xdp_vsi(vsi)) {
2911                 int frame_size = new_mtu + I40E_PACKET_HDR_PAD;
2912
2913                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2914                         return -EINVAL;
2915         }
2916
2917         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2918                    netdev->mtu, new_mtu);
2919         netdev->mtu = new_mtu;
2920         if (netif_running(netdev))
2921                 i40e_vsi_reinit_locked(vsi);
2922         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2923         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2924         return 0;
2925 }
2926
2927 /**
2928  * i40e_ioctl - Access the hwtstamp interface
2929  * @netdev: network interface device structure
2930  * @ifr: interface request data
2931  * @cmd: ioctl command
2932  **/
2933 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2934 {
2935         struct i40e_netdev_priv *np = netdev_priv(netdev);
2936         struct i40e_pf *pf = np->vsi->back;
2937
2938         switch (cmd) {
2939         case SIOCGHWTSTAMP:
2940                 return i40e_ptp_get_ts_config(pf, ifr);
2941         case SIOCSHWTSTAMP:
2942                 return i40e_ptp_set_ts_config(pf, ifr);
2943         default:
2944                 return -EOPNOTSUPP;
2945         }
2946 }
2947
2948 /**
2949  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2950  * @vsi: the vsi being adjusted
2951  **/
2952 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2953 {
2954         struct i40e_vsi_context ctxt;
2955         int ret;
2956
2957         /* Don't modify stripping options if a port VLAN is active */
2958         if (vsi->info.pvid)
2959                 return;
2960
2961         if ((vsi->info.valid_sections &
2962              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2963             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2964                 return;  /* already enabled */
2965
2966         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2967         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2968                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2969
2970         ctxt.seid = vsi->seid;
2971         ctxt.info = vsi->info;
2972         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2973         if (ret) {
2974                 dev_info(&vsi->back->pdev->dev,
2975                          "update vlan stripping failed, err %d aq_err %s\n",
2976                          ret,
2977                          i40e_aq_str(&vsi->back->hw,
2978                                      vsi->back->hw.aq.asq_last_status));
2979         }
2980 }
2981
2982 /**
2983  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2984  * @vsi: the vsi being adjusted
2985  **/
2986 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2987 {
2988         struct i40e_vsi_context ctxt;
2989         int ret;
2990
2991         /* Don't modify stripping options if a port VLAN is active */
2992         if (vsi->info.pvid)
2993                 return;
2994
2995         if ((vsi->info.valid_sections &
2996              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2997             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2998              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2999                 return;  /* already disabled */
3000
3001         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3002         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3003                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3004
3005         ctxt.seid = vsi->seid;
3006         ctxt.info = vsi->info;
3007         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3008         if (ret) {
3009                 dev_info(&vsi->back->pdev->dev,
3010                          "update vlan stripping failed, err %d aq_err %s\n",
3011                          ret,
3012                          i40e_aq_str(&vsi->back->hw,
3013                                      vsi->back->hw.aq.asq_last_status));
3014         }
3015 }
3016
3017 /**
3018  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3019  * @vsi: the vsi being configured
3020  * @vid: vlan id to be added (0 = untagged only , -1 = any)
3021  *
3022  * This is a helper function for adding a new MAC/VLAN filter with the
3023  * specified VLAN for each existing MAC address already in the hash table.
3024  * This function does *not* perform any accounting to update filters based on
3025  * VLAN mode.
3026  *
3027  * NOTE: this function expects to be called while under the
3028  * mac_filter_hash_lock
3029  **/
3030 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3031 {
3032         struct i40e_mac_filter *f, *add_f;
3033         struct hlist_node *h;
3034         int bkt;
3035
3036         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3037                 /* If we're asked to add a filter that has been marked for
3038                  * removal, it is safe to simply restore it to active state.
3039                  * __i40e_del_filter will have simply deleted any filters which
3040                  * were previously marked NEW or FAILED, so if it is currently
3041                  * marked REMOVE it must have previously been ACTIVE. Since we
3042                  * haven't yet run the sync filters task, just restore this
3043                  * filter to the ACTIVE state so that the sync task leaves it
3044                  * in place.
3045                  */
3046                 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3047                         f->state = I40E_FILTER_ACTIVE;
3048                         continue;
3049                 } else if (f->state == I40E_FILTER_REMOVE) {
3050                         continue;
3051                 }
3052                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
3053                 if (!add_f) {
3054                         dev_info(&vsi->back->pdev->dev,
3055                                  "Could not add vlan filter %d for %pM\n",
3056                                  vid, f->macaddr);
3057                         return -ENOMEM;
3058                 }
3059         }
3060
3061         return 0;
3062 }
3063
3064 /**
3065  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3066  * @vsi: the VSI being configured
3067  * @vid: VLAN id to be added
3068  **/
3069 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3070 {
3071         int err;
3072
3073         if (vsi->info.pvid)
3074                 return -EINVAL;
3075
3076         /* The network stack will attempt to add VID=0, with the intention to
3077          * receive priority tagged packets with a VLAN of 0. Our HW receives
3078          * these packets by default when configured to receive untagged
3079          * packets, so we don't need to add a filter for this case.
3080          * Additionally, HW interprets adding a VID=0 filter as meaning to
3081          * receive *only* tagged traffic and stops receiving untagged traffic.
3082          * Thus, we do not want to actually add a filter for VID=0
3083          */
3084         if (!vid)
3085                 return 0;
3086
3087         /* Locked once because all functions invoked below iterates list*/
3088         spin_lock_bh(&vsi->mac_filter_hash_lock);
3089         err = i40e_add_vlan_all_mac(vsi, vid);
3090         spin_unlock_bh(&vsi->mac_filter_hash_lock);
3091         if (err)
3092                 return err;
3093
3094         /* schedule our worker thread which will take care of
3095          * applying the new filter changes
3096          */
3097         i40e_service_event_schedule(vsi->back);
3098         return 0;
3099 }
3100
3101 /**
3102  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3103  * @vsi: the vsi being configured
3104  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3105  *
3106  * This function should be used to remove all VLAN filters which match the
3107  * given VID. It does not schedule the service event and does not take the
3108  * mac_filter_hash_lock so it may be combined with other operations under
3109  * a single invocation of the mac_filter_hash_lock.
3110  *
3111  * NOTE: this function expects to be called while under the
3112  * mac_filter_hash_lock
3113  */
3114 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3115 {
3116         struct i40e_mac_filter *f;
3117         struct hlist_node *h;
3118         int bkt;
3119
3120         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3121                 if (f->vlan == vid)
3122                         __i40e_del_filter(vsi, f);
3123         }
3124 }
3125
3126 /**
3127  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3128  * @vsi: the VSI being configured
3129  * @vid: VLAN id to be removed
3130  **/
3131 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3132 {
3133         if (!vid || vsi->info.pvid)
3134                 return;
3135
3136         spin_lock_bh(&vsi->mac_filter_hash_lock);
3137         i40e_rm_vlan_all_mac(vsi, vid);
3138         spin_unlock_bh(&vsi->mac_filter_hash_lock);
3139
3140         /* schedule our worker thread which will take care of
3141          * applying the new filter changes
3142          */
3143         i40e_service_event_schedule(vsi->back);
3144 }
3145
3146 /**
3147  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3148  * @netdev: network interface to be adjusted
3149  * @proto: unused protocol value
3150  * @vid: vlan id to be added
3151  *
3152  * net_device_ops implementation for adding vlan ids
3153  **/
3154 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3155                                 __always_unused __be16 proto, u16 vid)
3156 {
3157         struct i40e_netdev_priv *np = netdev_priv(netdev);
3158         struct i40e_vsi *vsi = np->vsi;
3159         int ret = 0;
3160
3161         if (vid >= VLAN_N_VID)
3162                 return -EINVAL;
3163
3164         ret = i40e_vsi_add_vlan(vsi, vid);
3165         if (!ret)
3166                 set_bit(vid, vsi->active_vlans);
3167
3168         return ret;
3169 }
3170
3171 /**
3172  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3173  * @netdev: network interface to be adjusted
3174  * @proto: unused protocol value
3175  * @vid: vlan id to be added
3176  **/
3177 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3178                                     __always_unused __be16 proto, u16 vid)
3179 {
3180         struct i40e_netdev_priv *np = netdev_priv(netdev);
3181         struct i40e_vsi *vsi = np->vsi;
3182
3183         if (vid >= VLAN_N_VID)
3184                 return;
3185         set_bit(vid, vsi->active_vlans);
3186 }
3187
3188 /**
3189  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3190  * @netdev: network interface to be adjusted
3191  * @proto: unused protocol value
3192  * @vid: vlan id to be removed
3193  *
3194  * net_device_ops implementation for removing vlan ids
3195  **/
3196 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3197                                  __always_unused __be16 proto, u16 vid)
3198 {
3199         struct i40e_netdev_priv *np = netdev_priv(netdev);
3200         struct i40e_vsi *vsi = np->vsi;
3201
3202         /* return code is ignored as there is nothing a user
3203          * can do about failure to remove and a log message was
3204          * already printed from the other function
3205          */
3206         i40e_vsi_kill_vlan(vsi, vid);
3207
3208         clear_bit(vid, vsi->active_vlans);
3209
3210         return 0;
3211 }
3212
3213 /**
3214  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3215  * @vsi: the vsi being brought back up
3216  **/
3217 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3218 {
3219         u16 vid;
3220
3221         if (!vsi->netdev)
3222                 return;
3223
3224         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3225                 i40e_vlan_stripping_enable(vsi);
3226         else
3227                 i40e_vlan_stripping_disable(vsi);
3228
3229         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3230                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3231                                         vid);
3232 }
3233
3234 /**
3235  * i40e_vsi_add_pvid - Add pvid for the VSI
3236  * @vsi: the vsi being adjusted
3237  * @vid: the vlan id to set as a PVID
3238  **/
3239 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3240 {
3241         struct i40e_vsi_context ctxt;
3242         int ret;
3243
3244         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3245         vsi->info.pvid = cpu_to_le16(vid);
3246         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3247                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
3248                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
3249
3250         ctxt.seid = vsi->seid;
3251         ctxt.info = vsi->info;
3252         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3253         if (ret) {
3254                 dev_info(&vsi->back->pdev->dev,
3255                          "add pvid failed, err %d aq_err %s\n",
3256                          ret,
3257                          i40e_aq_str(&vsi->back->hw,
3258                                      vsi->back->hw.aq.asq_last_status));
3259                 return -ENOENT;
3260         }
3261
3262         return 0;
3263 }
3264
3265 /**
3266  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3267  * @vsi: the vsi being adjusted
3268  *
3269  * Just use the vlan_rx_register() service to put it back to normal
3270  **/
3271 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3272 {
3273         vsi->info.pvid = 0;
3274
3275         i40e_vlan_stripping_disable(vsi);
3276 }
3277
3278 /**
3279  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3280  * @vsi: ptr to the VSI
3281  *
3282  * If this function returns with an error, then it's possible one or
3283  * more of the rings is populated (while the rest are not).  It is the
3284  * callers duty to clean those orphaned rings.
3285  *
3286  * Return 0 on success, negative on failure
3287  **/
3288 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3289 {
3290         int i, err = 0;
3291
3292         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3293                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3294
3295         if (!i40e_enabled_xdp_vsi(vsi))
3296                 return err;
3297
3298         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3299                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3300
3301         return err;
3302 }
3303
3304 /**
3305  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3306  * @vsi: ptr to the VSI
3307  *
3308  * Free VSI's transmit software resources
3309  **/
3310 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3311 {
3312         int i;
3313
3314         if (vsi->tx_rings) {
3315                 for (i = 0; i < vsi->num_queue_pairs; i++)
3316                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3317                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3318         }
3319
3320         if (vsi->xdp_rings) {
3321                 for (i = 0; i < vsi->num_queue_pairs; i++)
3322                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3323                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3324         }
3325 }
3326
3327 /**
3328  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3329  * @vsi: ptr to the VSI
3330  *
3331  * If this function returns with an error, then it's possible one or
3332  * more of the rings is populated (while the rest are not).  It is the
3333  * callers duty to clean those orphaned rings.
3334  *
3335  * Return 0 on success, negative on failure
3336  **/
3337 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3338 {
3339         int i, err = 0;
3340
3341         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3342                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3343         return err;
3344 }
3345
3346 /**
3347  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3348  * @vsi: ptr to the VSI
3349  *
3350  * Free all receive software resources
3351  **/
3352 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3353 {
3354         int i;
3355
3356         if (!vsi->rx_rings)
3357                 return;
3358
3359         for (i = 0; i < vsi->num_queue_pairs; i++)
3360                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3361                         i40e_free_rx_resources(vsi->rx_rings[i]);
3362 }
3363
3364 /**
3365  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3366  * @ring: The Tx ring to configure
3367  *
3368  * This enables/disables XPS for a given Tx descriptor ring
3369  * based on the TCs enabled for the VSI that ring belongs to.
3370  **/
3371 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3372 {
3373         int cpu;
3374
3375         if (!ring->q_vector || !ring->netdev || ring->ch)
3376                 return;
3377
3378         /* We only initialize XPS once, so as not to overwrite user settings */
3379         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3380                 return;
3381
3382         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3383         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3384                             ring->queue_index);
3385 }
3386
3387 /**
3388  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3389  * @ring: The Tx or Rx ring
3390  *
3391  * Returns the AF_XDP buffer pool or NULL.
3392  **/
3393 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3394 {
3395         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3396         int qid = ring->queue_index;
3397
3398         if (ring_is_xdp(ring))
3399                 qid -= ring->vsi->alloc_queue_pairs;
3400
3401         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3402                 return NULL;
3403
3404         return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3405 }
3406
3407 /**
3408  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3409  * @ring: The Tx ring to configure
3410  *
3411  * Configure the Tx descriptor ring in the HMC context.
3412  **/
3413 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3414 {
3415         struct i40e_vsi *vsi = ring->vsi;
3416         u16 pf_q = vsi->base_queue + ring->queue_index;
3417         struct i40e_hw *hw = &vsi->back->hw;
3418         struct i40e_hmc_obj_txq tx_ctx;
3419         u32 qtx_ctl = 0;
3420         int err = 0;
3421
3422         if (ring_is_xdp(ring))
3423                 ring->xsk_pool = i40e_xsk_pool(ring);
3424
3425         /* some ATR related tx ring init */
3426         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3427                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3428                 ring->atr_count = 0;
3429         } else {
3430                 ring->atr_sample_rate = 0;
3431         }
3432
3433         /* configure XPS */
3434         i40e_config_xps_tx_ring(ring);
3435
3436         /* clear the context structure first */
3437         memset(&tx_ctx, 0, sizeof(tx_ctx));
3438
3439         tx_ctx.new_context = 1;
3440         tx_ctx.base = (ring->dma / 128);
3441         tx_ctx.qlen = ring->count;
3442         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3443                                                I40E_FLAG_FD_ATR_ENABLED));
3444         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3445         /* FDIR VSI tx ring can still use RS bit and writebacks */
3446         if (vsi->type != I40E_VSI_FDIR)
3447                 tx_ctx.head_wb_ena = 1;
3448         tx_ctx.head_wb_addr = ring->dma +
3449                               (ring->count * sizeof(struct i40e_tx_desc));
3450
3451         /* As part of VSI creation/update, FW allocates certain
3452          * Tx arbitration queue sets for each TC enabled for
3453          * the VSI. The FW returns the handles to these queue
3454          * sets as part of the response buffer to Add VSI,
3455          * Update VSI, etc. AQ commands. It is expected that
3456          * these queue set handles be associated with the Tx
3457          * queues by the driver as part of the TX queue context
3458          * initialization. This has to be done regardless of
3459          * DCB as by default everything is mapped to TC0.
3460          */
3461
3462         if (ring->ch)
3463                 tx_ctx.rdylist =
3464                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3465
3466         else
3467                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3468
3469         tx_ctx.rdylist_act = 0;
3470
3471         /* clear the context in the HMC */
3472         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3473         if (err) {
3474                 dev_info(&vsi->back->pdev->dev,
3475                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3476                          ring->queue_index, pf_q, err);
3477                 return -ENOMEM;
3478         }
3479
3480         /* set the context in the HMC */
3481         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3482         if (err) {
3483                 dev_info(&vsi->back->pdev->dev,
3484                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3485                          ring->queue_index, pf_q, err);
3486                 return -ENOMEM;
3487         }
3488
3489         /* Now associate this queue with this PCI function */
3490         if (ring->ch) {
3491                 if (ring->ch->type == I40E_VSI_VMDQ2)
3492                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3493                 else
3494                         return -EINVAL;
3495
3496                 qtx_ctl |= (ring->ch->vsi_number <<
3497                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3498                             I40E_QTX_CTL_VFVM_INDX_MASK;
3499         } else {
3500                 if (vsi->type == I40E_VSI_VMDQ2) {
3501                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3502                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3503                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3504                 } else {
3505                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3506                 }
3507         }
3508
3509         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3510                     I40E_QTX_CTL_PF_INDX_MASK);
3511         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3512         i40e_flush(hw);
3513
3514         /* cache tail off for easier writes later */
3515         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3516
3517         return 0;
3518 }
3519
3520 /**
3521  * i40e_rx_offset - Return expected offset into page to access data
3522  * @rx_ring: Ring we are requesting offset of
3523  *
3524  * Returns the offset value for ring into the data buffer.
3525  */
3526 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3527 {
3528         return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3529 }
3530
3531 /**
3532  * i40e_configure_rx_ring - Configure a receive ring context
3533  * @ring: The Rx ring to configure
3534  *
3535  * Configure the Rx descriptor ring in the HMC context.
3536  **/
3537 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3538 {
3539         struct i40e_vsi *vsi = ring->vsi;
3540         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3541         u16 pf_q = vsi->base_queue + ring->queue_index;
3542         struct i40e_hw *hw = &vsi->back->hw;
3543         struct i40e_hmc_obj_rxq rx_ctx;
3544         int err = 0;
3545         bool ok;
3546         int ret;
3547
3548         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3549
3550         /* clear the context structure first */
3551         memset(&rx_ctx, 0, sizeof(rx_ctx));
3552
3553         if (ring->vsi->type == I40E_VSI_MAIN)
3554                 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3555
3556         ring->xsk_pool = i40e_xsk_pool(ring);
3557         if (ring->xsk_pool) {
3558                 ring->rx_buf_len =
3559                   xsk_pool_get_rx_frame_size(ring->xsk_pool);
3560                 /* For AF_XDP ZC, we disallow packets to span on
3561                  * multiple buffers, thus letting us skip that
3562                  * handling in the fast-path.
3563                  */
3564                 chain_len = 1;
3565                 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3566                                                  MEM_TYPE_XSK_BUFF_POOL,
3567                                                  NULL);
3568                 if (ret)
3569                         return ret;
3570                 dev_info(&vsi->back->pdev->dev,
3571                          "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3572                          ring->queue_index);
3573
3574         } else {
3575                 ring->rx_buf_len = vsi->rx_buf_len;
3576                 if (ring->vsi->type == I40E_VSI_MAIN) {
3577                         ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3578                                                          MEM_TYPE_PAGE_SHARED,
3579                                                          NULL);
3580                         if (ret)
3581                                 return ret;
3582                 }
3583         }
3584
3585         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3586                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3587
3588         rx_ctx.base = (ring->dma / 128);
3589         rx_ctx.qlen = ring->count;
3590
3591         /* use 16 byte descriptors */
3592         rx_ctx.dsize = 0;
3593
3594         /* descriptor type is always zero
3595          * rx_ctx.dtype = 0;
3596          */
3597         rx_ctx.hsplit_0 = 0;
3598
3599         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3600         if (hw->revision_id == 0)
3601                 rx_ctx.lrxqthresh = 0;
3602         else
3603                 rx_ctx.lrxqthresh = 1;
3604         rx_ctx.crcstrip = 1;
3605         rx_ctx.l2tsel = 1;
3606         /* this controls whether VLAN is stripped from inner headers */
3607         rx_ctx.showiv = 0;
3608         /* set the prefena field to 1 because the manual says to */
3609         rx_ctx.prefena = 1;
3610
3611         /* clear the context in the HMC */
3612         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3613         if (err) {
3614                 dev_info(&vsi->back->pdev->dev,
3615                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3616                          ring->queue_index, pf_q, err);
3617                 return -ENOMEM;
3618         }
3619
3620         /* set the context in the HMC */
3621         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3622         if (err) {
3623                 dev_info(&vsi->back->pdev->dev,
3624                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3625                          ring->queue_index, pf_q, err);
3626                 return -ENOMEM;
3627         }
3628
3629         /* configure Rx buffer alignment */
3630         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3631                 clear_ring_build_skb_enabled(ring);
3632         else
3633                 set_ring_build_skb_enabled(ring);
3634
3635         ring->rx_offset = i40e_rx_offset(ring);
3636
3637         /* cache tail for quicker writes, and clear the reg before use */
3638         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3639         writel(0, ring->tail);
3640
3641         if (ring->xsk_pool) {
3642                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3643                 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3644         } else {
3645                 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3646         }
3647         if (!ok) {
3648                 /* Log this in case the user has forgotten to give the kernel
3649                  * any buffers, even later in the application.
3650                  */
3651                 dev_info(&vsi->back->pdev->dev,
3652                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3653                          ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3654                          ring->queue_index, pf_q);
3655         }
3656
3657         return 0;
3658 }
3659
3660 /**
3661  * i40e_vsi_configure_tx - Configure the VSI for Tx
3662  * @vsi: VSI structure describing this set of rings and resources
3663  *
3664  * Configure the Tx VSI for operation.
3665  **/
3666 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3667 {
3668         int err = 0;
3669         u16 i;
3670
3671         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3672                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3673
3674         if (err || !i40e_enabled_xdp_vsi(vsi))
3675                 return err;
3676
3677         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3678                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3679
3680         return err;
3681 }
3682
3683 /**
3684  * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
3685  *
3686  * @vsi: VSI to calculate rx_buf_len from
3687  */
3688 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
3689 {
3690         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3691                 return I40E_RXBUFFER_2048;
3692
3693 #if (PAGE_SIZE < 8192)
3694         if (!I40E_2K_TOO_SMALL_WITH_PADDING && vsi->netdev->mtu <= ETH_DATA_LEN)
3695                 return I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3696 #endif
3697
3698         return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
3699 }
3700
3701 /**
3702  * i40e_vsi_configure_rx - Configure the VSI for Rx
3703  * @vsi: the VSI being configured
3704  *
3705  * Configure the Rx VSI for operation.
3706  **/
3707 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3708 {
3709         int err = 0;
3710         u16 i;
3711
3712         vsi->max_frame = I40E_MAX_RXBUFFER;
3713         vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3714
3715 #if (PAGE_SIZE < 8192)
3716         if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3717             vsi->netdev->mtu <= ETH_DATA_LEN)
3718                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3719 #endif
3720
3721         /* set up individual rings */
3722         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3723                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3724
3725         return err;
3726 }
3727
3728 /**
3729  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3730  * @vsi: ptr to the VSI
3731  **/
3732 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3733 {
3734         struct i40e_ring *tx_ring, *rx_ring;
3735         u16 qoffset, qcount;
3736         int i, n;
3737
3738         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3739                 /* Reset the TC information */
3740                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3741                         rx_ring = vsi->rx_rings[i];
3742                         tx_ring = vsi->tx_rings[i];
3743                         rx_ring->dcb_tc = 0;
3744                         tx_ring->dcb_tc = 0;
3745                 }
3746                 return;
3747         }
3748
3749         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3750                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3751                         continue;
3752
3753                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3754                 qcount = vsi->tc_config.tc_info[n].qcount;
3755                 for (i = qoffset; i < (qoffset + qcount); i++) {
3756                         rx_ring = vsi->rx_rings[i];
3757                         tx_ring = vsi->tx_rings[i];
3758                         rx_ring->dcb_tc = n;
3759                         tx_ring->dcb_tc = n;
3760                 }
3761         }
3762 }
3763
3764 /**
3765  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3766  * @vsi: ptr to the VSI
3767  **/
3768 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3769 {
3770         if (vsi->netdev)
3771                 i40e_set_rx_mode(vsi->netdev);
3772 }
3773
3774 /**
3775  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3776  * @pf: Pointer to the targeted PF
3777  *
3778  * Set all flow director counters to 0.
3779  */
3780 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3781 {
3782         pf->fd_tcp4_filter_cnt = 0;
3783         pf->fd_udp4_filter_cnt = 0;
3784         pf->fd_sctp4_filter_cnt = 0;
3785         pf->fd_ip4_filter_cnt = 0;
3786         pf->fd_tcp6_filter_cnt = 0;
3787         pf->fd_udp6_filter_cnt = 0;
3788         pf->fd_sctp6_filter_cnt = 0;
3789         pf->fd_ip6_filter_cnt = 0;
3790 }
3791
3792 /**
3793  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3794  * @vsi: Pointer to the targeted VSI
3795  *
3796  * This function replays the hlist on the hw where all the SB Flow Director
3797  * filters were saved.
3798  **/
3799 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3800 {
3801         struct i40e_fdir_filter *filter;
3802         struct i40e_pf *pf = vsi->back;
3803         struct hlist_node *node;
3804
3805         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3806                 return;
3807
3808         /* Reset FDir counters as we're replaying all existing filters */
3809         i40e_reset_fdir_filter_cnt(pf);
3810
3811         hlist_for_each_entry_safe(filter, node,
3812                                   &pf->fdir_filter_list, fdir_node) {
3813                 i40e_add_del_fdir(vsi, filter, true);
3814         }
3815 }
3816
3817 /**
3818  * i40e_vsi_configure - Set up the VSI for action
3819  * @vsi: the VSI being configured
3820  **/
3821 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3822 {
3823         int err;
3824
3825         i40e_set_vsi_rx_mode(vsi);
3826         i40e_restore_vlan(vsi);
3827         i40e_vsi_config_dcb_rings(vsi);
3828         err = i40e_vsi_configure_tx(vsi);
3829         if (!err)
3830                 err = i40e_vsi_configure_rx(vsi);
3831
3832         return err;
3833 }
3834
3835 /**
3836  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3837  * @vsi: the VSI being configured
3838  **/
3839 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3840 {
3841         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3842         struct i40e_pf *pf = vsi->back;
3843         struct i40e_hw *hw = &pf->hw;
3844         u16 vector;
3845         int i, q;
3846         u32 qp;
3847
3848         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3849          * and PFINT_LNKLSTn registers, e.g.:
3850          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3851          */
3852         qp = vsi->base_queue;
3853         vector = vsi->base_vector;
3854         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3855                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3856
3857                 q_vector->rx.next_update = jiffies + 1;
3858                 q_vector->rx.target_itr =
3859                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3860                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3861                      q_vector->rx.target_itr >> 1);
3862                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3863
3864                 q_vector->tx.next_update = jiffies + 1;
3865                 q_vector->tx.target_itr =
3866                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3867                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3868                      q_vector->tx.target_itr >> 1);
3869                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3870
3871                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3872                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3873
3874                 /* Linked list for the queuepairs assigned to this vector */
3875                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3876                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3877                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3878                         u32 val;
3879
3880                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3881                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3882                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3883                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3884                               (I40E_QUEUE_TYPE_TX <<
3885                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3886
3887                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3888
3889                         if (has_xdp) {
3890                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3891                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3892                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3893                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3894                                       (I40E_QUEUE_TYPE_TX <<
3895                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3896
3897                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3898                         }
3899
3900                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3901                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3902                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3903                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3904                               (I40E_QUEUE_TYPE_RX <<
3905                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3906
3907                         /* Terminate the linked list */
3908                         if (q == (q_vector->num_ringpairs - 1))
3909                                 val |= (I40E_QUEUE_END_OF_LIST <<
3910                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3911
3912                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3913                         qp++;
3914                 }
3915         }
3916
3917         i40e_flush(hw);
3918 }
3919
3920 /**
3921  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3922  * @pf: pointer to private device data structure
3923  **/
3924 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3925 {
3926         struct i40e_hw *hw = &pf->hw;
3927         u32 val;
3928
3929         /* clear things first */
3930         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3931         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3932
3933         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3934               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3935               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3936               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3937               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3938               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3939               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3940               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3941
3942         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3943                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3944
3945         if (pf->flags & I40E_FLAG_PTP)
3946                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3947
3948         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3949
3950         /* SW_ITR_IDX = 0, but don't change INTENA */
3951         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3952                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3953
3954         /* OTHER_ITR_IDX = 0 */
3955         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3956 }
3957
3958 /**
3959  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3960  * @vsi: the VSI being configured
3961  **/
3962 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3963 {
3964         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3965         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3966         struct i40e_pf *pf = vsi->back;
3967         struct i40e_hw *hw = &pf->hw;
3968         u32 val;
3969
3970         /* set the ITR configuration */
3971         q_vector->rx.next_update = jiffies + 1;
3972         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3973         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3974         q_vector->rx.current_itr = q_vector->rx.target_itr;
3975         q_vector->tx.next_update = jiffies + 1;
3976         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3977         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3978         q_vector->tx.current_itr = q_vector->tx.target_itr;
3979
3980         i40e_enable_misc_int_causes(pf);
3981
3982         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3983         wr32(hw, I40E_PFINT_LNKLST0, 0);
3984
3985         /* Associate the queue pair to the vector and enable the queue int */
3986         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3987               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3988               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3989               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3990
3991         wr32(hw, I40E_QINT_RQCTL(0), val);
3992
3993         if (i40e_enabled_xdp_vsi(vsi)) {
3994                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3995                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3996                       (I40E_QUEUE_TYPE_TX
3997                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3998
3999                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
4000         }
4001
4002         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
4003               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
4004               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
4005
4006         wr32(hw, I40E_QINT_TQCTL(0), val);
4007         i40e_flush(hw);
4008 }
4009
4010 /**
4011  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4012  * @pf: board private structure
4013  **/
4014 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4015 {
4016         struct i40e_hw *hw = &pf->hw;
4017
4018         wr32(hw, I40E_PFINT_DYN_CTL0,
4019              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4020         i40e_flush(hw);
4021 }
4022
4023 /**
4024  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4025  * @pf: board private structure
4026  **/
4027 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4028 {
4029         struct i40e_hw *hw = &pf->hw;
4030         u32 val;
4031
4032         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
4033               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4034               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4035
4036         wr32(hw, I40E_PFINT_DYN_CTL0, val);
4037         i40e_flush(hw);
4038 }
4039
4040 /**
4041  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4042  * @irq: interrupt number
4043  * @data: pointer to a q_vector
4044  **/
4045 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4046 {
4047         struct i40e_q_vector *q_vector = data;
4048
4049         if (!q_vector->tx.ring && !q_vector->rx.ring)
4050                 return IRQ_HANDLED;
4051
4052         napi_schedule_irqoff(&q_vector->napi);
4053
4054         return IRQ_HANDLED;
4055 }
4056
4057 /**
4058  * i40e_irq_affinity_notify - Callback for affinity changes
4059  * @notify: context as to what irq was changed
4060  * @mask: the new affinity mask
4061  *
4062  * This is a callback function used by the irq_set_affinity_notifier function
4063  * so that we may register to receive changes to the irq affinity masks.
4064  **/
4065 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4066                                      const cpumask_t *mask)
4067 {
4068         struct i40e_q_vector *q_vector =
4069                 container_of(notify, struct i40e_q_vector, affinity_notify);
4070
4071         cpumask_copy(&q_vector->affinity_mask, mask);
4072 }
4073
4074 /**
4075  * i40e_irq_affinity_release - Callback for affinity notifier release
4076  * @ref: internal core kernel usage
4077  *
4078  * This is a callback function used by the irq_set_affinity_notifier function
4079  * to inform the current notification subscriber that they will no longer
4080  * receive notifications.
4081  **/
4082 static void i40e_irq_affinity_release(struct kref *ref) {}
4083
4084 /**
4085  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4086  * @vsi: the VSI being configured
4087  * @basename: name for the vector
4088  *
4089  * Allocates MSI-X vectors and requests interrupts from the kernel.
4090  **/
4091 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4092 {
4093         int q_vectors = vsi->num_q_vectors;
4094         struct i40e_pf *pf = vsi->back;
4095         int base = vsi->base_vector;
4096         int rx_int_idx = 0;
4097         int tx_int_idx = 0;
4098         int vector, err;
4099         int irq_num;
4100         int cpu;
4101
4102         for (vector = 0; vector < q_vectors; vector++) {
4103                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4104
4105                 irq_num = pf->msix_entries[base + vector].vector;
4106
4107                 if (q_vector->tx.ring && q_vector->rx.ring) {
4108                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4109                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4110                         tx_int_idx++;
4111                 } else if (q_vector->rx.ring) {
4112                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4113                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
4114                 } else if (q_vector->tx.ring) {
4115                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4116                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
4117                 } else {
4118                         /* skip this unused q_vector */
4119                         continue;
4120                 }
4121                 err = request_irq(irq_num,
4122                                   vsi->irq_handler,
4123                                   0,
4124                                   q_vector->name,
4125                                   q_vector);
4126                 if (err) {
4127                         dev_info(&pf->pdev->dev,
4128                                  "MSIX request_irq failed, error: %d\n", err);
4129                         goto free_queue_irqs;
4130                 }
4131
4132                 /* register for affinity change notifications */
4133                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4134                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
4135                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4136                 /* Spread affinity hints out across online CPUs.
4137                  *
4138                  * get_cpu_mask returns a static constant mask with
4139                  * a permanent lifetime so it's ok to pass to
4140                  * irq_set_affinity_hint without making a copy.
4141                  */
4142                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
4143                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
4144         }
4145
4146         vsi->irqs_ready = true;
4147         return 0;
4148
4149 free_queue_irqs:
4150         while (vector) {
4151                 vector--;
4152                 irq_num = pf->msix_entries[base + vector].vector;
4153                 irq_set_affinity_notifier(irq_num, NULL);
4154                 irq_set_affinity_hint(irq_num, NULL);
4155                 free_irq(irq_num, &vsi->q_vectors[vector]);
4156         }
4157         return err;
4158 }
4159
4160 /**
4161  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4162  * @vsi: the VSI being un-configured
4163  **/
4164 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4165 {
4166         struct i40e_pf *pf = vsi->back;
4167         struct i40e_hw *hw = &pf->hw;
4168         int base = vsi->base_vector;
4169         int i;
4170
4171         /* disable interrupt causation from each queue */
4172         for (i = 0; i < vsi->num_queue_pairs; i++) {
4173                 u32 val;
4174
4175                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4176                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4177                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4178
4179                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4180                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4181                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4182
4183                 if (!i40e_enabled_xdp_vsi(vsi))
4184                         continue;
4185                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4186         }
4187
4188         /* disable each interrupt */
4189         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4190                 for (i = vsi->base_vector;
4191                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
4192                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4193
4194                 i40e_flush(hw);
4195                 for (i = 0; i < vsi->num_q_vectors; i++)
4196                         synchronize_irq(pf->msix_entries[i + base].vector);
4197         } else {
4198                 /* Legacy and MSI mode - this stops all interrupt handling */
4199                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4200                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4201                 i40e_flush(hw);
4202                 synchronize_irq(pf->pdev->irq);
4203         }
4204 }
4205
4206 /**
4207  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4208  * @vsi: the VSI being configured
4209  **/
4210 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4211 {
4212         struct i40e_pf *pf = vsi->back;
4213         int i;
4214
4215         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4216                 for (i = 0; i < vsi->num_q_vectors; i++)
4217                         i40e_irq_dynamic_enable(vsi, i);
4218         } else {
4219                 i40e_irq_dynamic_enable_icr0(pf);
4220         }
4221
4222         i40e_flush(&pf->hw);
4223         return 0;
4224 }
4225
4226 /**
4227  * i40e_free_misc_vector - Free the vector that handles non-queue events
4228  * @pf: board private structure
4229  **/
4230 static void i40e_free_misc_vector(struct i40e_pf *pf)
4231 {
4232         /* Disable ICR 0 */
4233         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4234         i40e_flush(&pf->hw);
4235
4236         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4237                 synchronize_irq(pf->msix_entries[0].vector);
4238                 free_irq(pf->msix_entries[0].vector, pf);
4239                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4240         }
4241 }
4242
4243 /**
4244  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4245  * @irq: interrupt number
4246  * @data: pointer to a q_vector
4247  *
4248  * This is the handler used for all MSI/Legacy interrupts, and deals
4249  * with both queue and non-queue interrupts.  This is also used in
4250  * MSIX mode to handle the non-queue interrupts.
4251  **/
4252 static irqreturn_t i40e_intr(int irq, void *data)
4253 {
4254         struct i40e_pf *pf = (struct i40e_pf *)data;
4255         struct i40e_hw *hw = &pf->hw;
4256         irqreturn_t ret = IRQ_NONE;
4257         u32 icr0, icr0_remaining;
4258         u32 val, ena_mask;
4259
4260         icr0 = rd32(hw, I40E_PFINT_ICR0);
4261         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4262
4263         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4264         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4265                 goto enable_intr;
4266
4267         /* if interrupt but no bits showing, must be SWINT */
4268         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4269             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4270                 pf->sw_int_count++;
4271
4272         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
4273             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4274                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4275                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4276                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4277         }
4278
4279         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4280         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4281                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4282                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4283
4284                 /* We do not have a way to disarm Queue causes while leaving
4285                  * interrupt enabled for all other causes, ideally
4286                  * interrupt should be disabled while we are in NAPI but
4287                  * this is not a performance path and napi_schedule()
4288                  * can deal with rescheduling.
4289                  */
4290                 if (!test_bit(__I40E_DOWN, pf->state))
4291                         napi_schedule_irqoff(&q_vector->napi);
4292         }
4293
4294         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4295                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4296                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4297                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4298         }
4299
4300         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4301                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4302                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4303         }
4304
4305         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4306                 /* disable any further VFLR event notifications */
4307                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4308                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4309
4310                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4311                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4312                 } else {
4313                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4314                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4315                 }
4316         }
4317
4318         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4319                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4320                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4321                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4322                 val = rd32(hw, I40E_GLGEN_RSTAT);
4323                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4324                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4325                 if (val == I40E_RESET_CORER) {
4326                         pf->corer_count++;
4327                 } else if (val == I40E_RESET_GLOBR) {
4328                         pf->globr_count++;
4329                 } else if (val == I40E_RESET_EMPR) {
4330                         pf->empr_count++;
4331                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4332                 }
4333         }
4334
4335         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4336                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4337                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4338                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4339                          rd32(hw, I40E_PFHMC_ERRORINFO),
4340                          rd32(hw, I40E_PFHMC_ERRORDATA));
4341         }
4342
4343         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4344                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4345
4346                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4347                         schedule_work(&pf->ptp_extts0_work);
4348
4349                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4350                         i40e_ptp_tx_hwtstamp(pf);
4351
4352                 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4353         }
4354
4355         /* If a critical error is pending we have no choice but to reset the
4356          * device.
4357          * Report and mask out any remaining unexpected interrupts.
4358          */
4359         icr0_remaining = icr0 & ena_mask;
4360         if (icr0_remaining) {
4361                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4362                          icr0_remaining);
4363                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4364                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4365                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4366                         dev_info(&pf->pdev->dev, "device will be reset\n");
4367                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4368                         i40e_service_event_schedule(pf);
4369                 }
4370                 ena_mask &= ~icr0_remaining;
4371         }
4372         ret = IRQ_HANDLED;
4373
4374 enable_intr:
4375         /* re-enable interrupt causes */
4376         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4377         if (!test_bit(__I40E_DOWN, pf->state) ||
4378             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4379                 i40e_service_event_schedule(pf);
4380                 i40e_irq_dynamic_enable_icr0(pf);
4381         }
4382
4383         return ret;
4384 }
4385
4386 /**
4387  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4388  * @tx_ring:  tx ring to clean
4389  * @budget:   how many cleans we're allowed
4390  *
4391  * Returns true if there's any budget left (e.g. the clean is finished)
4392  **/
4393 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4394 {
4395         struct i40e_vsi *vsi = tx_ring->vsi;
4396         u16 i = tx_ring->next_to_clean;
4397         struct i40e_tx_buffer *tx_buf;
4398         struct i40e_tx_desc *tx_desc;
4399
4400         tx_buf = &tx_ring->tx_bi[i];
4401         tx_desc = I40E_TX_DESC(tx_ring, i);
4402         i -= tx_ring->count;
4403
4404         do {
4405                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4406
4407                 /* if next_to_watch is not set then there is no work pending */
4408                 if (!eop_desc)
4409                         break;
4410
4411                 /* prevent any other reads prior to eop_desc */
4412                 smp_rmb();
4413
4414                 /* if the descriptor isn't done, no work yet to do */
4415                 if (!(eop_desc->cmd_type_offset_bsz &
4416                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4417                         break;
4418
4419                 /* clear next_to_watch to prevent false hangs */
4420                 tx_buf->next_to_watch = NULL;
4421
4422                 tx_desc->buffer_addr = 0;
4423                 tx_desc->cmd_type_offset_bsz = 0;
4424                 /* move past filter desc */
4425                 tx_buf++;
4426                 tx_desc++;
4427                 i++;
4428                 if (unlikely(!i)) {
4429                         i -= tx_ring->count;
4430                         tx_buf = tx_ring->tx_bi;
4431                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4432                 }
4433                 /* unmap skb header data */
4434                 dma_unmap_single(tx_ring->dev,
4435                                  dma_unmap_addr(tx_buf, dma),
4436                                  dma_unmap_len(tx_buf, len),
4437                                  DMA_TO_DEVICE);
4438                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4439                         kfree(tx_buf->raw_buf);
4440
4441                 tx_buf->raw_buf = NULL;
4442                 tx_buf->tx_flags = 0;
4443                 tx_buf->next_to_watch = NULL;
4444                 dma_unmap_len_set(tx_buf, len, 0);
4445                 tx_desc->buffer_addr = 0;
4446                 tx_desc->cmd_type_offset_bsz = 0;
4447
4448                 /* move us past the eop_desc for start of next FD desc */
4449                 tx_buf++;
4450                 tx_desc++;
4451                 i++;
4452                 if (unlikely(!i)) {
4453                         i -= tx_ring->count;
4454                         tx_buf = tx_ring->tx_bi;
4455                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4456                 }
4457
4458                 /* update budget accounting */
4459                 budget--;
4460         } while (likely(budget));
4461
4462         i += tx_ring->count;
4463         tx_ring->next_to_clean = i;
4464
4465         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4466                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4467
4468         return budget > 0;
4469 }
4470
4471 /**
4472  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4473  * @irq: interrupt number
4474  * @data: pointer to a q_vector
4475  **/
4476 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4477 {
4478         struct i40e_q_vector *q_vector = data;
4479         struct i40e_vsi *vsi;
4480
4481         if (!q_vector->tx.ring)
4482                 return IRQ_HANDLED;
4483
4484         vsi = q_vector->tx.ring->vsi;
4485         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4486
4487         return IRQ_HANDLED;
4488 }
4489
4490 /**
4491  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4492  * @vsi: the VSI being configured
4493  * @v_idx: vector index
4494  * @qp_idx: queue pair index
4495  **/
4496 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4497 {
4498         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4499         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4500         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4501
4502         tx_ring->q_vector = q_vector;
4503         tx_ring->next = q_vector->tx.ring;
4504         q_vector->tx.ring = tx_ring;
4505         q_vector->tx.count++;
4506
4507         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4508         if (i40e_enabled_xdp_vsi(vsi)) {
4509                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4510
4511                 xdp_ring->q_vector = q_vector;
4512                 xdp_ring->next = q_vector->tx.ring;
4513                 q_vector->tx.ring = xdp_ring;
4514                 q_vector->tx.count++;
4515         }
4516
4517         rx_ring->q_vector = q_vector;
4518         rx_ring->next = q_vector->rx.ring;
4519         q_vector->rx.ring = rx_ring;
4520         q_vector->rx.count++;
4521 }
4522
4523 /**
4524  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4525  * @vsi: the VSI being configured
4526  *
4527  * This function maps descriptor rings to the queue-specific vectors
4528  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4529  * one vector per queue pair, but on a constrained vector budget, we
4530  * group the queue pairs as "efficiently" as possible.
4531  **/
4532 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4533 {
4534         int qp_remaining = vsi->num_queue_pairs;
4535         int q_vectors = vsi->num_q_vectors;
4536         int num_ringpairs;
4537         int v_start = 0;
4538         int qp_idx = 0;
4539
4540         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4541          * group them so there are multiple queues per vector.
4542          * It is also important to go through all the vectors available to be
4543          * sure that if we don't use all the vectors, that the remaining vectors
4544          * are cleared. This is especially important when decreasing the
4545          * number of queues in use.
4546          */
4547         for (; v_start < q_vectors; v_start++) {
4548                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4549
4550                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4551
4552                 q_vector->num_ringpairs = num_ringpairs;
4553                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4554
4555                 q_vector->rx.count = 0;
4556                 q_vector->tx.count = 0;
4557                 q_vector->rx.ring = NULL;
4558                 q_vector->tx.ring = NULL;
4559
4560                 while (num_ringpairs--) {
4561                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4562                         qp_idx++;
4563                         qp_remaining--;
4564                 }
4565         }
4566 }
4567
4568 /**
4569  * i40e_vsi_request_irq - Request IRQ from the OS
4570  * @vsi: the VSI being configured
4571  * @basename: name for the vector
4572  **/
4573 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4574 {
4575         struct i40e_pf *pf = vsi->back;
4576         int err;
4577
4578         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4579                 err = i40e_vsi_request_irq_msix(vsi, basename);
4580         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4581                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4582                                   pf->int_name, pf);
4583         else
4584                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4585                                   pf->int_name, pf);
4586
4587         if (err)
4588                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4589
4590         return err;
4591 }
4592
4593 #ifdef CONFIG_NET_POLL_CONTROLLER
4594 /**
4595  * i40e_netpoll - A Polling 'interrupt' handler
4596  * @netdev: network interface device structure
4597  *
4598  * This is used by netconsole to send skbs without having to re-enable
4599  * interrupts.  It's not called while the normal interrupt routine is executing.
4600  **/
4601 static void i40e_netpoll(struct net_device *netdev)
4602 {
4603         struct i40e_netdev_priv *np = netdev_priv(netdev);
4604         struct i40e_vsi *vsi = np->vsi;
4605         struct i40e_pf *pf = vsi->back;
4606         int i;
4607
4608         /* if interface is down do nothing */
4609         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4610                 return;
4611
4612         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4613                 for (i = 0; i < vsi->num_q_vectors; i++)
4614                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4615         } else {
4616                 i40e_intr(pf->pdev->irq, netdev);
4617         }
4618 }
4619 #endif
4620
4621 #define I40E_QTX_ENA_WAIT_COUNT 50
4622
4623 /**
4624  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4625  * @pf: the PF being configured
4626  * @pf_q: the PF queue
4627  * @enable: enable or disable state of the queue
4628  *
4629  * This routine will wait for the given Tx queue of the PF to reach the
4630  * enabled or disabled state.
4631  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4632  * multiple retries; else will return 0 in case of success.
4633  **/
4634 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4635 {
4636         int i;
4637         u32 tx_reg;
4638
4639         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4640                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4641                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4642                         break;
4643
4644                 usleep_range(10, 20);
4645         }
4646         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4647                 return -ETIMEDOUT;
4648
4649         return 0;
4650 }
4651
4652 /**
4653  * i40e_control_tx_q - Start or stop a particular Tx queue
4654  * @pf: the PF structure
4655  * @pf_q: the PF queue to configure
4656  * @enable: start or stop the queue
4657  *
4658  * This function enables or disables a single queue. Note that any delay
4659  * required after the operation is expected to be handled by the caller of
4660  * this function.
4661  **/
4662 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4663 {
4664         struct i40e_hw *hw = &pf->hw;
4665         u32 tx_reg;
4666         int i;
4667
4668         /* warn the TX unit of coming changes */
4669         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4670         if (!enable)
4671                 usleep_range(10, 20);
4672
4673         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4674                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4675                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4676                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4677                         break;
4678                 usleep_range(1000, 2000);
4679         }
4680
4681         /* Skip if the queue is already in the requested state */
4682         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4683                 return;
4684
4685         /* turn on/off the queue */
4686         if (enable) {
4687                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4688                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4689         } else {
4690                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4691         }
4692
4693         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4694 }
4695
4696 /**
4697  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4698  * @seid: VSI SEID
4699  * @pf: the PF structure
4700  * @pf_q: the PF queue to configure
4701  * @is_xdp: true if the queue is used for XDP
4702  * @enable: start or stop the queue
4703  **/
4704 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4705                            bool is_xdp, bool enable)
4706 {
4707         int ret;
4708
4709         i40e_control_tx_q(pf, pf_q, enable);
4710
4711         /* wait for the change to finish */
4712         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4713         if (ret) {
4714                 dev_info(&pf->pdev->dev,
4715                          "VSI seid %d %sTx ring %d %sable timeout\n",
4716                          seid, (is_xdp ? "XDP " : ""), pf_q,
4717                          (enable ? "en" : "dis"));
4718         }
4719
4720         return ret;
4721 }
4722
4723 /**
4724  * i40e_vsi_enable_tx - Start a VSI's rings
4725  * @vsi: the VSI being configured
4726  **/
4727 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4728 {
4729         struct i40e_pf *pf = vsi->back;
4730         int i, pf_q, ret = 0;
4731
4732         pf_q = vsi->base_queue;
4733         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4734                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4735                                              pf_q,
4736                                              false /*is xdp*/, true);
4737                 if (ret)
4738                         break;
4739
4740                 if (!i40e_enabled_xdp_vsi(vsi))
4741                         continue;
4742
4743                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4744                                              pf_q + vsi->alloc_queue_pairs,
4745                                              true /*is xdp*/, true);
4746                 if (ret)
4747                         break;
4748         }
4749         return ret;
4750 }
4751
4752 /**
4753  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4754  * @pf: the PF being configured
4755  * @pf_q: the PF queue
4756  * @enable: enable or disable state of the queue
4757  *
4758  * This routine will wait for the given Rx queue of the PF to reach the
4759  * enabled or disabled state.
4760  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4761  * multiple retries; else will return 0 in case of success.
4762  **/
4763 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4764 {
4765         int i;
4766         u32 rx_reg;
4767
4768         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4769                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4770                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4771                         break;
4772
4773                 usleep_range(10, 20);
4774         }
4775         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4776                 return -ETIMEDOUT;
4777
4778         return 0;
4779 }
4780
4781 /**
4782  * i40e_control_rx_q - Start or stop a particular Rx queue
4783  * @pf: the PF structure
4784  * @pf_q: the PF queue to configure
4785  * @enable: start or stop the queue
4786  *
4787  * This function enables or disables a single queue. Note that
4788  * any delay required after the operation is expected to be
4789  * handled by the caller of this function.
4790  **/
4791 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4792 {
4793         struct i40e_hw *hw = &pf->hw;
4794         u32 rx_reg;
4795         int i;
4796
4797         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4798                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4799                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4800                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4801                         break;
4802                 usleep_range(1000, 2000);
4803         }
4804
4805         /* Skip if the queue is already in the requested state */
4806         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4807                 return;
4808
4809         /* turn on/off the queue */
4810         if (enable)
4811                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4812         else
4813                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4814
4815         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4816 }
4817
4818 /**
4819  * i40e_control_wait_rx_q
4820  * @pf: the PF structure
4821  * @pf_q: queue being configured
4822  * @enable: start or stop the rings
4823  *
4824  * This function enables or disables a single queue along with waiting
4825  * for the change to finish. The caller of this function should handle
4826  * the delays needed in the case of disabling queues.
4827  **/
4828 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4829 {
4830         int ret = 0;
4831
4832         i40e_control_rx_q(pf, pf_q, enable);
4833
4834         /* wait for the change to finish */
4835         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4836         if (ret)
4837                 return ret;
4838
4839         return ret;
4840 }
4841
4842 /**
4843  * i40e_vsi_enable_rx - Start a VSI's rings
4844  * @vsi: the VSI being configured
4845  **/
4846 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4847 {
4848         struct i40e_pf *pf = vsi->back;
4849         int i, pf_q, ret = 0;
4850
4851         pf_q = vsi->base_queue;
4852         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4853                 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4854                 if (ret) {
4855                         dev_info(&pf->pdev->dev,
4856                                  "VSI seid %d Rx ring %d enable timeout\n",
4857                                  vsi->seid, pf_q);
4858                         break;
4859                 }
4860         }
4861
4862         return ret;
4863 }
4864
4865 /**
4866  * i40e_vsi_start_rings - Start a VSI's rings
4867  * @vsi: the VSI being configured
4868  **/
4869 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4870 {
4871         int ret = 0;
4872
4873         /* do rx first for enable and last for disable */
4874         ret = i40e_vsi_enable_rx(vsi);
4875         if (ret)
4876                 return ret;
4877         ret = i40e_vsi_enable_tx(vsi);
4878
4879         return ret;
4880 }
4881
4882 #define I40E_DISABLE_TX_GAP_MSEC        50
4883
4884 /**
4885  * i40e_vsi_stop_rings - Stop a VSI's rings
4886  * @vsi: the VSI being configured
4887  **/
4888 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4889 {
4890         struct i40e_pf *pf = vsi->back;
4891         int pf_q, err, q_end;
4892
4893         /* When port TX is suspended, don't wait */
4894         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4895                 return i40e_vsi_stop_rings_no_wait(vsi);
4896
4897         q_end = vsi->base_queue + vsi->num_queue_pairs;
4898         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4899                 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4900
4901         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4902                 err = i40e_control_wait_rx_q(pf, pf_q, false);
4903                 if (err)
4904                         dev_info(&pf->pdev->dev,
4905                                  "VSI seid %d Rx ring %d disable timeout\n",
4906                                  vsi->seid, pf_q);
4907         }
4908
4909         msleep(I40E_DISABLE_TX_GAP_MSEC);
4910         pf_q = vsi->base_queue;
4911         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4912                 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4913
4914         i40e_vsi_wait_queues_disabled(vsi);
4915 }
4916
4917 /**
4918  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4919  * @vsi: the VSI being shutdown
4920  *
4921  * This function stops all the rings for a VSI but does not delay to verify
4922  * that rings have been disabled. It is expected that the caller is shutting
4923  * down multiple VSIs at once and will delay together for all the VSIs after
4924  * initiating the shutdown. This is particularly useful for shutting down lots
4925  * of VFs together. Otherwise, a large delay can be incurred while configuring
4926  * each VSI in serial.
4927  **/
4928 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4929 {
4930         struct i40e_pf *pf = vsi->back;
4931         int i, pf_q;
4932
4933         pf_q = vsi->base_queue;
4934         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4935                 i40e_control_tx_q(pf, pf_q, false);
4936                 i40e_control_rx_q(pf, pf_q, false);
4937         }
4938 }
4939
4940 /**
4941  * i40e_vsi_free_irq - Free the irq association with the OS
4942  * @vsi: the VSI being configured
4943  **/
4944 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4945 {
4946         struct i40e_pf *pf = vsi->back;
4947         struct i40e_hw *hw = &pf->hw;
4948         int base = vsi->base_vector;
4949         u32 val, qp;
4950         int i;
4951
4952         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4953                 if (!vsi->q_vectors)
4954                         return;
4955
4956                 if (!vsi->irqs_ready)
4957                         return;
4958
4959                 vsi->irqs_ready = false;
4960                 for (i = 0; i < vsi->num_q_vectors; i++) {
4961                         int irq_num;
4962                         u16 vector;
4963
4964                         vector = i + base;
4965                         irq_num = pf->msix_entries[vector].vector;
4966
4967                         /* free only the irqs that were actually requested */
4968                         if (!vsi->q_vectors[i] ||
4969                             !vsi->q_vectors[i]->num_ringpairs)
4970                                 continue;
4971
4972                         /* clear the affinity notifier in the IRQ descriptor */
4973                         irq_set_affinity_notifier(irq_num, NULL);
4974                         /* remove our suggested affinity mask for this IRQ */
4975                         irq_set_affinity_hint(irq_num, NULL);
4976                         synchronize_irq(irq_num);
4977                         free_irq(irq_num, vsi->q_vectors[i]);
4978
4979                         /* Tear down the interrupt queue link list
4980                          *
4981                          * We know that they come in pairs and always
4982                          * the Rx first, then the Tx.  To clear the
4983                          * link list, stick the EOL value into the
4984                          * next_q field of the registers.
4985                          */
4986                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4987                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4988                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4989                         val |= I40E_QUEUE_END_OF_LIST
4990                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4991                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4992
4993                         while (qp != I40E_QUEUE_END_OF_LIST) {
4994                                 u32 next;
4995
4996                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4997
4998                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4999                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5000                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5001                                          I40E_QINT_RQCTL_INTEVENT_MASK);
5002
5003                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5004                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5005
5006                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
5007
5008                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
5009
5010                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
5011                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
5012
5013                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5014                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5015                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5016                                          I40E_QINT_TQCTL_INTEVENT_MASK);
5017
5018                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5019                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5020
5021                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
5022                                 qp = next;
5023                         }
5024                 }
5025         } else {
5026                 free_irq(pf->pdev->irq, pf);
5027
5028                 val = rd32(hw, I40E_PFINT_LNKLST0);
5029                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
5030                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5031                 val |= I40E_QUEUE_END_OF_LIST
5032                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5033                 wr32(hw, I40E_PFINT_LNKLST0, val);
5034
5035                 val = rd32(hw, I40E_QINT_RQCTL(qp));
5036                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5037                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5038                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5039                          I40E_QINT_RQCTL_INTEVENT_MASK);
5040
5041                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5042                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5043
5044                 wr32(hw, I40E_QINT_RQCTL(qp), val);
5045
5046                 val = rd32(hw, I40E_QINT_TQCTL(qp));
5047
5048                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5049                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5050                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5051                          I40E_QINT_TQCTL_INTEVENT_MASK);
5052
5053                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5054                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5055
5056                 wr32(hw, I40E_QINT_TQCTL(qp), val);
5057         }
5058 }
5059
5060 /**
5061  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5062  * @vsi: the VSI being configured
5063  * @v_idx: Index of vector to be freed
5064  *
5065  * This function frees the memory allocated to the q_vector.  In addition if
5066  * NAPI is enabled it will delete any references to the NAPI struct prior
5067  * to freeing the q_vector.
5068  **/
5069 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5070 {
5071         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5072         struct i40e_ring *ring;
5073
5074         if (!q_vector)
5075                 return;
5076
5077         /* disassociate q_vector from rings */
5078         i40e_for_each_ring(ring, q_vector->tx)
5079                 ring->q_vector = NULL;
5080
5081         i40e_for_each_ring(ring, q_vector->rx)
5082                 ring->q_vector = NULL;
5083
5084         /* only VSI w/ an associated netdev is set up w/ NAPI */
5085         if (vsi->netdev)
5086                 netif_napi_del(&q_vector->napi);
5087
5088         vsi->q_vectors[v_idx] = NULL;
5089
5090         kfree_rcu(q_vector, rcu);
5091 }
5092
5093 /**
5094  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5095  * @vsi: the VSI being un-configured
5096  *
5097  * This frees the memory allocated to the q_vectors and
5098  * deletes references to the NAPI struct.
5099  **/
5100 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5101 {
5102         int v_idx;
5103
5104         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5105                 i40e_free_q_vector(vsi, v_idx);
5106 }
5107
5108 /**
5109  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5110  * @pf: board private structure
5111  **/
5112 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5113 {
5114         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5115         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
5116                 pci_disable_msix(pf->pdev);
5117                 kfree(pf->msix_entries);
5118                 pf->msix_entries = NULL;
5119                 kfree(pf->irq_pile);
5120                 pf->irq_pile = NULL;
5121         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
5122                 pci_disable_msi(pf->pdev);
5123         }
5124         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
5125 }
5126
5127 /**
5128  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5129  * @pf: board private structure
5130  *
5131  * We go through and clear interrupt specific resources and reset the structure
5132  * to pre-load conditions
5133  **/
5134 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5135 {
5136         int i;
5137
5138         if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5139                 i40e_free_misc_vector(pf);
5140
5141         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5142                       I40E_IWARP_IRQ_PILE_ID);
5143
5144         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5145         for (i = 0; i < pf->num_alloc_vsi; i++)
5146                 if (pf->vsi[i])
5147                         i40e_vsi_free_q_vectors(pf->vsi[i]);
5148         i40e_reset_interrupt_capability(pf);
5149 }
5150
5151 /**
5152  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5153  * @vsi: the VSI being configured
5154  **/
5155 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5156 {
5157         int q_idx;
5158
5159         if (!vsi->netdev)
5160                 return;
5161
5162         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5163                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5164
5165                 if (q_vector->rx.ring || q_vector->tx.ring)
5166                         napi_enable(&q_vector->napi);
5167         }
5168 }
5169
5170 /**
5171  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5172  * @vsi: the VSI being configured
5173  **/
5174 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5175 {
5176         int q_idx;
5177
5178         if (!vsi->netdev)
5179                 return;
5180
5181         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5182                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5183
5184                 if (q_vector->rx.ring || q_vector->tx.ring)
5185                         napi_disable(&q_vector->napi);
5186         }
5187 }
5188
5189 /**
5190  * i40e_vsi_close - Shut down a VSI
5191  * @vsi: the vsi to be quelled
5192  **/
5193 static void i40e_vsi_close(struct i40e_vsi *vsi)
5194 {
5195         struct i40e_pf *pf = vsi->back;
5196         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5197                 i40e_down(vsi);
5198         i40e_vsi_free_irq(vsi);
5199         i40e_vsi_free_tx_resources(vsi);
5200         i40e_vsi_free_rx_resources(vsi);
5201         vsi->current_netdev_flags = 0;
5202         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5203         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5204                 set_bit(__I40E_CLIENT_RESET, pf->state);
5205 }
5206
5207 /**
5208  * i40e_quiesce_vsi - Pause a given VSI
5209  * @vsi: the VSI being paused
5210  **/
5211 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5212 {
5213         if (test_bit(__I40E_VSI_DOWN, vsi->state))
5214                 return;
5215
5216         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5217         if (vsi->netdev && netif_running(vsi->netdev))
5218                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5219         else
5220                 i40e_vsi_close(vsi);
5221 }
5222
5223 /**
5224  * i40e_unquiesce_vsi - Resume a given VSI
5225  * @vsi: the VSI being resumed
5226  **/
5227 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5228 {
5229         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5230                 return;
5231
5232         if (vsi->netdev && netif_running(vsi->netdev))
5233                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5234         else
5235                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
5236 }
5237
5238 /**
5239  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5240  * @pf: the PF
5241  **/
5242 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5243 {
5244         int v;
5245
5246         for (v = 0; v < pf->num_alloc_vsi; v++) {
5247                 if (pf->vsi[v])
5248                         i40e_quiesce_vsi(pf->vsi[v]);
5249         }
5250 }
5251
5252 /**
5253  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5254  * @pf: the PF
5255  **/
5256 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5257 {
5258         int v;
5259
5260         for (v = 0; v < pf->num_alloc_vsi; v++) {
5261                 if (pf->vsi[v])
5262                         i40e_unquiesce_vsi(pf->vsi[v]);
5263         }
5264 }
5265
5266 /**
5267  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5268  * @vsi: the VSI being configured
5269  *
5270  * Wait until all queues on a given VSI have been disabled.
5271  **/
5272 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5273 {
5274         struct i40e_pf *pf = vsi->back;
5275         int i, pf_q, ret;
5276
5277         pf_q = vsi->base_queue;
5278         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5279                 /* Check and wait for the Tx queue */
5280                 ret = i40e_pf_txq_wait(pf, pf_q, false);
5281                 if (ret) {
5282                         dev_info(&pf->pdev->dev,
5283                                  "VSI seid %d Tx ring %d disable timeout\n",
5284                                  vsi->seid, pf_q);
5285                         return ret;
5286                 }
5287
5288                 if (!i40e_enabled_xdp_vsi(vsi))
5289                         goto wait_rx;
5290
5291                 /* Check and wait for the XDP Tx queue */
5292                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5293                                        false);
5294                 if (ret) {
5295                         dev_info(&pf->pdev->dev,
5296                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
5297                                  vsi->seid, pf_q);
5298                         return ret;
5299                 }
5300 wait_rx:
5301                 /* Check and wait for the Rx queue */
5302                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5303                 if (ret) {
5304                         dev_info(&pf->pdev->dev,
5305                                  "VSI seid %d Rx ring %d disable timeout\n",
5306                                  vsi->seid, pf_q);
5307                         return ret;
5308                 }
5309         }
5310
5311         return 0;
5312 }
5313
5314 #ifdef CONFIG_I40E_DCB
5315 /**
5316  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5317  * @pf: the PF
5318  *
5319  * This function waits for the queues to be in disabled state for all the
5320  * VSIs that are managed by this PF.
5321  **/
5322 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5323 {
5324         int v, ret = 0;
5325
5326         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5327                 if (pf->vsi[v]) {
5328                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5329                         if (ret)
5330                                 break;
5331                 }
5332         }
5333
5334         return ret;
5335 }
5336
5337 #endif
5338
5339 /**
5340  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5341  * @pf: pointer to PF
5342  *
5343  * Get TC map for ISCSI PF type that will include iSCSI TC
5344  * and LAN TC.
5345  **/
5346 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5347 {
5348         struct i40e_dcb_app_priority_table app;
5349         struct i40e_hw *hw = &pf->hw;
5350         u8 enabled_tc = 1; /* TC0 is always enabled */
5351         u8 tc, i;
5352         /* Get the iSCSI APP TLV */
5353         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5354
5355         for (i = 0; i < dcbcfg->numapps; i++) {
5356                 app = dcbcfg->app[i];
5357                 if (app.selector == I40E_APP_SEL_TCPIP &&
5358                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5359                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5360                         enabled_tc |= BIT(tc);
5361                         break;
5362                 }
5363         }
5364
5365         return enabled_tc;
5366 }
5367
5368 /**
5369  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5370  * @dcbcfg: the corresponding DCBx configuration structure
5371  *
5372  * Return the number of TCs from given DCBx configuration
5373  **/
5374 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5375 {
5376         int i, tc_unused = 0;
5377         u8 num_tc = 0;
5378         u8 ret = 0;
5379
5380         /* Scan the ETS Config Priority Table to find
5381          * traffic class enabled for a given priority
5382          * and create a bitmask of enabled TCs
5383          */
5384         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5385                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5386
5387         /* Now scan the bitmask to check for
5388          * contiguous TCs starting with TC0
5389          */
5390         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5391                 if (num_tc & BIT(i)) {
5392                         if (!tc_unused) {
5393                                 ret++;
5394                         } else {
5395                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5396                                 return 1;
5397                         }
5398                 } else {
5399                         tc_unused = 1;
5400                 }
5401         }
5402
5403         /* There is always at least TC0 */
5404         if (!ret)
5405                 ret = 1;
5406
5407         return ret;
5408 }
5409
5410 /**
5411  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5412  * @dcbcfg: the corresponding DCBx configuration structure
5413  *
5414  * Query the current DCB configuration and return the number of
5415  * traffic classes enabled from the given DCBX config
5416  **/
5417 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5418 {
5419         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5420         u8 enabled_tc = 1;
5421         u8 i;
5422
5423         for (i = 0; i < num_tc; i++)
5424                 enabled_tc |= BIT(i);
5425
5426         return enabled_tc;
5427 }
5428
5429 /**
5430  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5431  * @pf: PF being queried
5432  *
5433  * Query the current MQPRIO configuration and return the number of
5434  * traffic classes enabled.
5435  **/
5436 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5437 {
5438         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5439         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5440         u8 enabled_tc = 1, i;
5441
5442         for (i = 1; i < num_tc; i++)
5443                 enabled_tc |= BIT(i);
5444         return enabled_tc;
5445 }
5446
5447 /**
5448  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5449  * @pf: PF being queried
5450  *
5451  * Return number of traffic classes enabled for the given PF
5452  **/
5453 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5454 {
5455         struct i40e_hw *hw = &pf->hw;
5456         u8 i, enabled_tc = 1;
5457         u8 num_tc = 0;
5458         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5459
5460         if (i40e_is_tc_mqprio_enabled(pf))
5461                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5462
5463         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5464         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5465                 return 1;
5466
5467         /* SFP mode will be enabled for all TCs on port */
5468         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5469                 return i40e_dcb_get_num_tc(dcbcfg);
5470
5471         /* MFP mode return count of enabled TCs for this PF */
5472         if (pf->hw.func_caps.iscsi)
5473                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5474         else
5475                 return 1; /* Only TC0 */
5476
5477         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5478                 if (enabled_tc & BIT(i))
5479                         num_tc++;
5480         }
5481         return num_tc;
5482 }
5483
5484 /**
5485  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5486  * @pf: PF being queried
5487  *
5488  * Return a bitmap for enabled traffic classes for this PF.
5489  **/
5490 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5491 {
5492         if (i40e_is_tc_mqprio_enabled(pf))
5493                 return i40e_mqprio_get_enabled_tc(pf);
5494
5495         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5496          * default TC
5497          */
5498         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5499                 return I40E_DEFAULT_TRAFFIC_CLASS;
5500
5501         /* SFP mode we want PF to be enabled for all TCs */
5502         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5503                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5504
5505         /* MFP enabled and iSCSI PF type */
5506         if (pf->hw.func_caps.iscsi)
5507                 return i40e_get_iscsi_tc_map(pf);
5508         else
5509                 return I40E_DEFAULT_TRAFFIC_CLASS;
5510 }
5511
5512 /**
5513  * i40e_vsi_get_bw_info - Query VSI BW Information
5514  * @vsi: the VSI being queried
5515  *
5516  * Returns 0 on success, negative value on failure
5517  **/
5518 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5519 {
5520         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5521         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5522         struct i40e_pf *pf = vsi->back;
5523         struct i40e_hw *hw = &pf->hw;
5524         u32 tc_bw_max;
5525         int ret;
5526         int i;
5527
5528         /* Get the VSI level BW configuration */
5529         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5530         if (ret) {
5531                 dev_info(&pf->pdev->dev,
5532                          "couldn't get PF vsi bw config, err %d aq_err %s\n",
5533                          ret,
5534                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5535                 return -EINVAL;
5536         }
5537
5538         /* Get the VSI level BW configuration per TC */
5539         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5540                                                NULL);
5541         if (ret) {
5542                 dev_info(&pf->pdev->dev,
5543                          "couldn't get PF vsi ets bw config, err %d aq_err %s\n",
5544                          ret,
5545                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5546                 return -EINVAL;
5547         }
5548
5549         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5550                 dev_info(&pf->pdev->dev,
5551                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5552                          bw_config.tc_valid_bits,
5553                          bw_ets_config.tc_valid_bits);
5554                 /* Still continuing */
5555         }
5556
5557         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5558         vsi->bw_max_quanta = bw_config.max_bw;
5559         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5560                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5561         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5562                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5563                 vsi->bw_ets_limit_credits[i] =
5564                                         le16_to_cpu(bw_ets_config.credits[i]);
5565                 /* 3 bits out of 4 for each TC */
5566                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5567         }
5568
5569         return 0;
5570 }
5571
5572 /**
5573  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5574  * @vsi: the VSI being configured
5575  * @enabled_tc: TC bitmap
5576  * @bw_share: BW shared credits per TC
5577  *
5578  * Returns 0 on success, negative value on failure
5579  **/
5580 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5581                                        u8 *bw_share)
5582 {
5583         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5584         struct i40e_pf *pf = vsi->back;
5585         int ret;
5586         int i;
5587
5588         /* There is no need to reset BW when mqprio mode is on.  */
5589         if (i40e_is_tc_mqprio_enabled(pf))
5590                 return 0;
5591         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5592                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5593                 if (ret)
5594                         dev_info(&pf->pdev->dev,
5595                                  "Failed to reset tx rate for vsi->seid %u\n",
5596                                  vsi->seid);
5597                 return ret;
5598         }
5599         memset(&bw_data, 0, sizeof(bw_data));
5600         bw_data.tc_valid_bits = enabled_tc;
5601         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5602                 bw_data.tc_bw_credits[i] = bw_share[i];
5603
5604         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5605         if (ret) {
5606                 dev_info(&pf->pdev->dev,
5607                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5608                          pf->hw.aq.asq_last_status);
5609                 return -EINVAL;
5610         }
5611
5612         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5613                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5614
5615         return 0;
5616 }
5617
5618 /**
5619  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5620  * @vsi: the VSI being configured
5621  * @enabled_tc: TC map to be enabled
5622  *
5623  **/
5624 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5625 {
5626         struct net_device *netdev = vsi->netdev;
5627         struct i40e_pf *pf = vsi->back;
5628         struct i40e_hw *hw = &pf->hw;
5629         u8 netdev_tc = 0;
5630         int i;
5631         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5632
5633         if (!netdev)
5634                 return;
5635
5636         if (!enabled_tc) {
5637                 netdev_reset_tc(netdev);
5638                 return;
5639         }
5640
5641         /* Set up actual enabled TCs on the VSI */
5642         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5643                 return;
5644
5645         /* set per TC queues for the VSI */
5646         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5647                 /* Only set TC queues for enabled tcs
5648                  *
5649                  * e.g. For a VSI that has TC0 and TC3 enabled the
5650                  * enabled_tc bitmap would be 0x00001001; the driver
5651                  * will set the numtc for netdev as 2 that will be
5652                  * referenced by the netdev layer as TC 0 and 1.
5653                  */
5654                 if (vsi->tc_config.enabled_tc & BIT(i))
5655                         netdev_set_tc_queue(netdev,
5656                                         vsi->tc_config.tc_info[i].netdev_tc,
5657                                         vsi->tc_config.tc_info[i].qcount,
5658                                         vsi->tc_config.tc_info[i].qoffset);
5659         }
5660
5661         if (i40e_is_tc_mqprio_enabled(pf))
5662                 return;
5663
5664         /* Assign UP2TC map for the VSI */
5665         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5666                 /* Get the actual TC# for the UP */
5667                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5668                 /* Get the mapped netdev TC# for the UP */
5669                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5670                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5671         }
5672 }
5673
5674 /**
5675  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5676  * @vsi: the VSI being configured
5677  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5678  **/
5679 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5680                                       struct i40e_vsi_context *ctxt)
5681 {
5682         /* copy just the sections touched not the entire info
5683          * since not all sections are valid as returned by
5684          * update vsi params
5685          */
5686         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5687         memcpy(&vsi->info.queue_mapping,
5688                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5689         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5690                sizeof(vsi->info.tc_mapping));
5691 }
5692
5693 /**
5694  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5695  * @vsi: the VSI being reconfigured
5696  * @vsi_offset: offset from main VF VSI
5697  */
5698 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5699 {
5700         struct i40e_vsi_context ctxt = {};
5701         struct i40e_pf *pf;
5702         struct i40e_hw *hw;
5703         int ret;
5704
5705         if (!vsi)
5706                 return I40E_ERR_PARAM;
5707         pf = vsi->back;
5708         hw = &pf->hw;
5709
5710         ctxt.seid = vsi->seid;
5711         ctxt.pf_num = hw->pf_id;
5712         ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5713         ctxt.uplink_seid = vsi->uplink_seid;
5714         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5715         ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5716         ctxt.info = vsi->info;
5717
5718         i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5719                                  false);
5720         if (vsi->reconfig_rss) {
5721                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5722                                       vsi->num_queue_pairs);
5723                 ret = i40e_vsi_config_rss(vsi);
5724                 if (ret) {
5725                         dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5726                         return ret;
5727                 }
5728                 vsi->reconfig_rss = false;
5729         }
5730
5731         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5732         if (ret) {
5733                 dev_info(&pf->pdev->dev, "Update vsi config failed, err %d aq_err %s\n",
5734                          ret,
5735                          i40e_aq_str(hw, hw->aq.asq_last_status));
5736                 return ret;
5737         }
5738         /* update the local VSI info with updated queue map */
5739         i40e_vsi_update_queue_map(vsi, &ctxt);
5740         vsi->info.valid_sections = 0;
5741
5742         return ret;
5743 }
5744
5745 /**
5746  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5747  * @vsi: VSI to be configured
5748  * @enabled_tc: TC bitmap
5749  *
5750  * This configures a particular VSI for TCs that are mapped to the
5751  * given TC bitmap. It uses default bandwidth share for TCs across
5752  * VSIs to configure TC for a particular VSI.
5753  *
5754  * NOTE:
5755  * It is expected that the VSI queues have been quisced before calling
5756  * this function.
5757  **/
5758 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5759 {
5760         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5761         struct i40e_pf *pf = vsi->back;
5762         struct i40e_hw *hw = &pf->hw;
5763         struct i40e_vsi_context ctxt;
5764         int ret = 0;
5765         int i;
5766
5767         /* Check if enabled_tc is same as existing or new TCs */
5768         if (vsi->tc_config.enabled_tc == enabled_tc &&
5769             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5770                 return ret;
5771
5772         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5773         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5774                 if (enabled_tc & BIT(i))
5775                         bw_share[i] = 1;
5776         }
5777
5778         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5779         if (ret) {
5780                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5781
5782                 dev_info(&pf->pdev->dev,
5783                          "Failed configuring TC map %d for VSI %d\n",
5784                          enabled_tc, vsi->seid);
5785                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5786                                                   &bw_config, NULL);
5787                 if (ret) {
5788                         dev_info(&pf->pdev->dev,
5789                                  "Failed querying vsi bw info, err %d aq_err %s\n",
5790                                  ret,
5791                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5792                         goto out;
5793                 }
5794                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5795                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5796
5797                         if (!valid_tc)
5798                                 valid_tc = bw_config.tc_valid_bits;
5799                         /* Always enable TC0, no matter what */
5800                         valid_tc |= 1;
5801                         dev_info(&pf->pdev->dev,
5802                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5803                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5804                         enabled_tc = valid_tc;
5805                 }
5806
5807                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5808                 if (ret) {
5809                         dev_err(&pf->pdev->dev,
5810                                 "Unable to  configure TC map %d for VSI %d\n",
5811                                 enabled_tc, vsi->seid);
5812                         goto out;
5813                 }
5814         }
5815
5816         /* Update Queue Pairs Mapping for currently enabled UPs */
5817         ctxt.seid = vsi->seid;
5818         ctxt.pf_num = vsi->back->hw.pf_id;
5819         ctxt.vf_num = 0;
5820         ctxt.uplink_seid = vsi->uplink_seid;
5821         ctxt.info = vsi->info;
5822         if (i40e_is_tc_mqprio_enabled(pf)) {
5823                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5824                 if (ret)
5825                         goto out;
5826         } else {
5827                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5828         }
5829
5830         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5831          * queues changed.
5832          */
5833         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5834                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5835                                       vsi->num_queue_pairs);
5836                 ret = i40e_vsi_config_rss(vsi);
5837                 if (ret) {
5838                         dev_info(&vsi->back->pdev->dev,
5839                                  "Failed to reconfig rss for num_queues\n");
5840                         return ret;
5841                 }
5842                 vsi->reconfig_rss = false;
5843         }
5844         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5845                 ctxt.info.valid_sections |=
5846                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5847                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5848         }
5849
5850         /* Update the VSI after updating the VSI queue-mapping
5851          * information
5852          */
5853         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5854         if (ret) {
5855                 dev_info(&pf->pdev->dev,
5856                          "Update vsi tc config failed, err %d aq_err %s\n",
5857                          ret,
5858                          i40e_aq_str(hw, hw->aq.asq_last_status));
5859                 goto out;
5860         }
5861         /* update the local VSI info with updated queue map */
5862         i40e_vsi_update_queue_map(vsi, &ctxt);
5863         vsi->info.valid_sections = 0;
5864
5865         /* Update current VSI BW information */
5866         ret = i40e_vsi_get_bw_info(vsi);
5867         if (ret) {
5868                 dev_info(&pf->pdev->dev,
5869                          "Failed updating vsi bw info, err %d aq_err %s\n",
5870                          ret,
5871                          i40e_aq_str(hw, hw->aq.asq_last_status));
5872                 goto out;
5873         }
5874
5875         /* Update the netdev TC setup */
5876         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5877 out:
5878         return ret;
5879 }
5880
5881 /**
5882  * i40e_get_link_speed - Returns link speed for the interface
5883  * @vsi: VSI to be configured
5884  *
5885  **/
5886 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5887 {
5888         struct i40e_pf *pf = vsi->back;
5889
5890         switch (pf->hw.phy.link_info.link_speed) {
5891         case I40E_LINK_SPEED_40GB:
5892                 return 40000;
5893         case I40E_LINK_SPEED_25GB:
5894                 return 25000;
5895         case I40E_LINK_SPEED_20GB:
5896                 return 20000;
5897         case I40E_LINK_SPEED_10GB:
5898                 return 10000;
5899         case I40E_LINK_SPEED_1GB:
5900                 return 1000;
5901         default:
5902                 return -EINVAL;
5903         }
5904 }
5905
5906 /**
5907  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5908  * @vsi: Pointer to vsi structure
5909  * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5910  *
5911  * Helper function to convert units before send to set BW limit
5912  **/
5913 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5914 {
5915         if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5916                 dev_warn(&vsi->back->pdev->dev,
5917                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5918                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5919         } else {
5920                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5921         }
5922
5923         return max_tx_rate;
5924 }
5925
5926 /**
5927  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5928  * @vsi: VSI to be configured
5929  * @seid: seid of the channel/VSI
5930  * @max_tx_rate: max TX rate to be configured as BW limit
5931  *
5932  * Helper function to set BW limit for a given VSI
5933  **/
5934 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5935 {
5936         struct i40e_pf *pf = vsi->back;
5937         u64 credits = 0;
5938         int speed = 0;
5939         int ret = 0;
5940
5941         speed = i40e_get_link_speed(vsi);
5942         if (max_tx_rate > speed) {
5943                 dev_err(&pf->pdev->dev,
5944                         "Invalid max tx rate %llu specified for VSI seid %d.",
5945                         max_tx_rate, seid);
5946                 return -EINVAL;
5947         }
5948         if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5949                 dev_warn(&pf->pdev->dev,
5950                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5951                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5952         }
5953
5954         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5955         credits = max_tx_rate;
5956         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5957         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5958                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5959         if (ret)
5960                 dev_err(&pf->pdev->dev,
5961                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %d aq_err %s\n",
5962                         max_tx_rate, seid, ret,
5963                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5964         return ret;
5965 }
5966
5967 /**
5968  * i40e_remove_queue_channels - Remove queue channels for the TCs
5969  * @vsi: VSI to be configured
5970  *
5971  * Remove queue channels for the TCs
5972  **/
5973 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5974 {
5975         enum i40e_admin_queue_err last_aq_status;
5976         struct i40e_cloud_filter *cfilter;
5977         struct i40e_channel *ch, *ch_tmp;
5978         struct i40e_pf *pf = vsi->back;
5979         struct hlist_node *node;
5980         int ret, i;
5981
5982         /* Reset rss size that was stored when reconfiguring rss for
5983          * channel VSIs with non-power-of-2 queue count.
5984          */
5985         vsi->current_rss_size = 0;
5986
5987         /* perform cleanup for channels if they exist */
5988         if (list_empty(&vsi->ch_list))
5989                 return;
5990
5991         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5992                 struct i40e_vsi *p_vsi;
5993
5994                 list_del(&ch->list);
5995                 p_vsi = ch->parent_vsi;
5996                 if (!p_vsi || !ch->initialized) {
5997                         kfree(ch);
5998                         continue;
5999                 }
6000                 /* Reset queue contexts */
6001                 for (i = 0; i < ch->num_queue_pairs; i++) {
6002                         struct i40e_ring *tx_ring, *rx_ring;
6003                         u16 pf_q;
6004
6005                         pf_q = ch->base_queue + i;
6006                         tx_ring = vsi->tx_rings[pf_q];
6007                         tx_ring->ch = NULL;
6008
6009                         rx_ring = vsi->rx_rings[pf_q];
6010                         rx_ring->ch = NULL;
6011                 }
6012
6013                 /* Reset BW configured for this VSI via mqprio */
6014                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6015                 if (ret)
6016                         dev_info(&vsi->back->pdev->dev,
6017                                  "Failed to reset tx rate for ch->seid %u\n",
6018                                  ch->seid);
6019
6020                 /* delete cloud filters associated with this channel */
6021                 hlist_for_each_entry_safe(cfilter, node,
6022                                           &pf->cloud_filter_list, cloud_node) {
6023                         if (cfilter->seid != ch->seid)
6024                                 continue;
6025
6026                         hash_del(&cfilter->cloud_node);
6027                         if (cfilter->dst_port)
6028                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
6029                                                                         cfilter,
6030                                                                         false);
6031                         else
6032                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
6033                                                                 false);
6034                         last_aq_status = pf->hw.aq.asq_last_status;
6035                         if (ret)
6036                                 dev_info(&pf->pdev->dev,
6037                                          "Failed to delete cloud filter, err %d aq_err %s\n",
6038                                          ret,
6039                                          i40e_aq_str(&pf->hw, last_aq_status));
6040                         kfree(cfilter);
6041                 }
6042
6043                 /* delete VSI from FW */
6044                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6045                                              NULL);
6046                 if (ret)
6047                         dev_err(&vsi->back->pdev->dev,
6048                                 "unable to remove channel (%d) for parent VSI(%d)\n",
6049                                 ch->seid, p_vsi->seid);
6050                 kfree(ch);
6051         }
6052         INIT_LIST_HEAD(&vsi->ch_list);
6053 }
6054
6055 /**
6056  * i40e_get_max_queues_for_channel
6057  * @vsi: ptr to VSI to which channels are associated with
6058  *
6059  * Helper function which returns max value among the queue counts set on the
6060  * channels/TCs created.
6061  **/
6062 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6063 {
6064         struct i40e_channel *ch, *ch_tmp;
6065         int max = 0;
6066
6067         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6068                 if (!ch->initialized)
6069                         continue;
6070                 if (ch->num_queue_pairs > max)
6071                         max = ch->num_queue_pairs;
6072         }
6073
6074         return max;
6075 }
6076
6077 /**
6078  * i40e_validate_num_queues - validate num_queues w.r.t channel
6079  * @pf: ptr to PF device
6080  * @num_queues: number of queues
6081  * @vsi: the parent VSI
6082  * @reconfig_rss: indicates should the RSS be reconfigured or not
6083  *
6084  * This function validates number of queues in the context of new channel
6085  * which is being established and determines if RSS should be reconfigured
6086  * or not for parent VSI.
6087  **/
6088 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6089                                     struct i40e_vsi *vsi, bool *reconfig_rss)
6090 {
6091         int max_ch_queues;
6092
6093         if (!reconfig_rss)
6094                 return -EINVAL;
6095
6096         *reconfig_rss = false;
6097         if (vsi->current_rss_size) {
6098                 if (num_queues > vsi->current_rss_size) {
6099                         dev_dbg(&pf->pdev->dev,
6100                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
6101                                 num_queues, vsi->current_rss_size);
6102                         return -EINVAL;
6103                 } else if ((num_queues < vsi->current_rss_size) &&
6104                            (!is_power_of_2(num_queues))) {
6105                         dev_dbg(&pf->pdev->dev,
6106                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6107                                 num_queues, vsi->current_rss_size);
6108                         return -EINVAL;
6109                 }
6110         }
6111
6112         if (!is_power_of_2(num_queues)) {
6113                 /* Find the max num_queues configured for channel if channel
6114                  * exist.
6115                  * if channel exist, then enforce 'num_queues' to be more than
6116                  * max ever queues configured for channel.
6117                  */
6118                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6119                 if (num_queues < max_ch_queues) {
6120                         dev_dbg(&pf->pdev->dev,
6121                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
6122                                 num_queues, max_ch_queues);
6123                         return -EINVAL;
6124                 }
6125                 *reconfig_rss = true;
6126         }
6127
6128         return 0;
6129 }
6130
6131 /**
6132  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6133  * @vsi: the VSI being setup
6134  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6135  *
6136  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6137  **/
6138 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6139 {
6140         struct i40e_pf *pf = vsi->back;
6141         u8 seed[I40E_HKEY_ARRAY_SIZE];
6142         struct i40e_hw *hw = &pf->hw;
6143         int local_rss_size;
6144         u8 *lut;
6145         int ret;
6146
6147         if (!vsi->rss_size)
6148                 return -EINVAL;
6149
6150         if (rss_size > vsi->rss_size)
6151                 return -EINVAL;
6152
6153         local_rss_size = min_t(int, vsi->rss_size, rss_size);
6154         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6155         if (!lut)
6156                 return -ENOMEM;
6157
6158         /* Ignoring user configured lut if there is one */
6159         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6160
6161         /* Use user configured hash key if there is one, otherwise
6162          * use default.
6163          */
6164         if (vsi->rss_hkey_user)
6165                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6166         else
6167                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6168
6169         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6170         if (ret) {
6171                 dev_info(&pf->pdev->dev,
6172                          "Cannot set RSS lut, err %d aq_err %s\n",
6173                          ret,
6174                          i40e_aq_str(hw, hw->aq.asq_last_status));
6175                 kfree(lut);
6176                 return ret;
6177         }
6178         kfree(lut);
6179
6180         /* Do the update w.r.t. storing rss_size */
6181         if (!vsi->orig_rss_size)
6182                 vsi->orig_rss_size = vsi->rss_size;
6183         vsi->current_rss_size = local_rss_size;
6184
6185         return ret;
6186 }
6187
6188 /**
6189  * i40e_channel_setup_queue_map - Setup a channel queue map
6190  * @pf: ptr to PF device
6191  * @ctxt: VSI context structure
6192  * @ch: ptr to channel structure
6193  *
6194  * Setup queue map for a specific channel
6195  **/
6196 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6197                                          struct i40e_vsi_context *ctxt,
6198                                          struct i40e_channel *ch)
6199 {
6200         u16 qcount, qmap, sections = 0;
6201         u8 offset = 0;
6202         int pow;
6203
6204         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6205         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6206
6207         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6208         ch->num_queue_pairs = qcount;
6209
6210         /* find the next higher power-of-2 of num queue pairs */
6211         pow = ilog2(qcount);
6212         if (!is_power_of_2(qcount))
6213                 pow++;
6214
6215         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6216                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6217
6218         /* Setup queue TC[0].qmap for given VSI context */
6219         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6220
6221         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6222         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6223         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6224         ctxt->info.valid_sections |= cpu_to_le16(sections);
6225 }
6226
6227 /**
6228  * i40e_add_channel - add a channel by adding VSI
6229  * @pf: ptr to PF device
6230  * @uplink_seid: underlying HW switching element (VEB) ID
6231  * @ch: ptr to channel structure
6232  *
6233  * Add a channel (VSI) using add_vsi and queue_map
6234  **/
6235 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6236                             struct i40e_channel *ch)
6237 {
6238         struct i40e_hw *hw = &pf->hw;
6239         struct i40e_vsi_context ctxt;
6240         u8 enabled_tc = 0x1; /* TC0 enabled */
6241         int ret;
6242
6243         if (ch->type != I40E_VSI_VMDQ2) {
6244                 dev_info(&pf->pdev->dev,
6245                          "add new vsi failed, ch->type %d\n", ch->type);
6246                 return -EINVAL;
6247         }
6248
6249         memset(&ctxt, 0, sizeof(ctxt));
6250         ctxt.pf_num = hw->pf_id;
6251         ctxt.vf_num = 0;
6252         ctxt.uplink_seid = uplink_seid;
6253         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6254         if (ch->type == I40E_VSI_VMDQ2)
6255                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6256
6257         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
6258                 ctxt.info.valid_sections |=
6259                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6260                 ctxt.info.switch_id =
6261                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6262         }
6263
6264         /* Set queue map for a given VSI context */
6265         i40e_channel_setup_queue_map(pf, &ctxt, ch);
6266
6267         /* Now time to create VSI */
6268         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6269         if (ret) {
6270                 dev_info(&pf->pdev->dev,
6271                          "add new vsi failed, err %d aq_err %s\n",
6272                          ret,
6273                          i40e_aq_str(&pf->hw,
6274                                      pf->hw.aq.asq_last_status));
6275                 return -ENOENT;
6276         }
6277
6278         /* Success, update channel, set enabled_tc only if the channel
6279          * is not a macvlan
6280          */
6281         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6282         ch->seid = ctxt.seid;
6283         ch->vsi_number = ctxt.vsi_number;
6284         ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6285
6286         /* copy just the sections touched not the entire info
6287          * since not all sections are valid as returned by
6288          * update vsi params
6289          */
6290         ch->info.mapping_flags = ctxt.info.mapping_flags;
6291         memcpy(&ch->info.queue_mapping,
6292                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6293         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6294                sizeof(ctxt.info.tc_mapping));
6295
6296         return 0;
6297 }
6298
6299 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6300                                   u8 *bw_share)
6301 {
6302         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6303         int ret;
6304         int i;
6305
6306         memset(&bw_data, 0, sizeof(bw_data));
6307         bw_data.tc_valid_bits = ch->enabled_tc;
6308         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6309                 bw_data.tc_bw_credits[i] = bw_share[i];
6310
6311         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6312                                        &bw_data, NULL);
6313         if (ret) {
6314                 dev_info(&vsi->back->pdev->dev,
6315                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6316                          vsi->back->hw.aq.asq_last_status, ch->seid);
6317                 return -EINVAL;
6318         }
6319
6320         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6321                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6322
6323         return 0;
6324 }
6325
6326 /**
6327  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6328  * @pf: ptr to PF device
6329  * @vsi: the VSI being setup
6330  * @ch: ptr to channel structure
6331  *
6332  * Configure TX rings associated with channel (VSI) since queues are being
6333  * from parent VSI.
6334  **/
6335 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6336                                        struct i40e_vsi *vsi,
6337                                        struct i40e_channel *ch)
6338 {
6339         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6340         int ret;
6341         int i;
6342
6343         /* Enable ETS TCs with equal BW Share for now across all VSIs */
6344         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6345                 if (ch->enabled_tc & BIT(i))
6346                         bw_share[i] = 1;
6347         }
6348
6349         /* configure BW for new VSI */
6350         ret = i40e_channel_config_bw(vsi, ch, bw_share);
6351         if (ret) {
6352                 dev_info(&vsi->back->pdev->dev,
6353                          "Failed configuring TC map %d for channel (seid %u)\n",
6354                          ch->enabled_tc, ch->seid);
6355                 return ret;
6356         }
6357
6358         for (i = 0; i < ch->num_queue_pairs; i++) {
6359                 struct i40e_ring *tx_ring, *rx_ring;
6360                 u16 pf_q;
6361
6362                 pf_q = ch->base_queue + i;
6363
6364                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6365                  * context
6366                  */
6367                 tx_ring = vsi->tx_rings[pf_q];
6368                 tx_ring->ch = ch;
6369
6370                 /* Get the RX ring ptr */
6371                 rx_ring = vsi->rx_rings[pf_q];
6372                 rx_ring->ch = ch;
6373         }
6374
6375         return 0;
6376 }
6377
6378 /**
6379  * i40e_setup_hw_channel - setup new channel
6380  * @pf: ptr to PF device
6381  * @vsi: the VSI being setup
6382  * @ch: ptr to channel structure
6383  * @uplink_seid: underlying HW switching element (VEB) ID
6384  * @type: type of channel to be created (VMDq2/VF)
6385  *
6386  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6387  * and configures TX rings accordingly
6388  **/
6389 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6390                                         struct i40e_vsi *vsi,
6391                                         struct i40e_channel *ch,
6392                                         u16 uplink_seid, u8 type)
6393 {
6394         int ret;
6395
6396         ch->initialized = false;
6397         ch->base_queue = vsi->next_base_queue;
6398         ch->type = type;
6399
6400         /* Proceed with creation of channel (VMDq2) VSI */
6401         ret = i40e_add_channel(pf, uplink_seid, ch);
6402         if (ret) {
6403                 dev_info(&pf->pdev->dev,
6404                          "failed to add_channel using uplink_seid %u\n",
6405                          uplink_seid);
6406                 return ret;
6407         }
6408
6409         /* Mark the successful creation of channel */
6410         ch->initialized = true;
6411
6412         /* Reconfigure TX queues using QTX_CTL register */
6413         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6414         if (ret) {
6415                 dev_info(&pf->pdev->dev,
6416                          "failed to configure TX rings for channel %u\n",
6417                          ch->seid);
6418                 return ret;
6419         }
6420
6421         /* update 'next_base_queue' */
6422         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6423         dev_dbg(&pf->pdev->dev,
6424                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6425                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6426                 ch->num_queue_pairs,
6427                 vsi->next_base_queue);
6428         return ret;
6429 }
6430
6431 /**
6432  * i40e_setup_channel - setup new channel using uplink element
6433  * @pf: ptr to PF device
6434  * @vsi: pointer to the VSI to set up the channel within
6435  * @ch: ptr to channel structure
6436  *
6437  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6438  * and uplink switching element (uplink_seid)
6439  **/
6440 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6441                                struct i40e_channel *ch)
6442 {
6443         u8 vsi_type;
6444         u16 seid;
6445         int ret;
6446
6447         if (vsi->type == I40E_VSI_MAIN) {
6448                 vsi_type = I40E_VSI_VMDQ2;
6449         } else {
6450                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6451                         vsi->type);
6452                 return false;
6453         }
6454
6455         /* underlying switching element */
6456         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6457
6458         /* create channel (VSI), configure TX rings */
6459         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6460         if (ret) {
6461                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6462                 return false;
6463         }
6464
6465         return ch->initialized ? true : false;
6466 }
6467
6468 /**
6469  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6470  * @vsi: ptr to VSI which has PF backing
6471  *
6472  * Sets up switch mode correctly if it needs to be changed and perform
6473  * what are allowed modes.
6474  **/
6475 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6476 {
6477         u8 mode;
6478         struct i40e_pf *pf = vsi->back;
6479         struct i40e_hw *hw = &pf->hw;
6480         int ret;
6481
6482         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6483         if (ret)
6484                 return -EINVAL;
6485
6486         if (hw->dev_caps.switch_mode) {
6487                 /* if switch mode is set, support mode2 (non-tunneled for
6488                  * cloud filter) for now
6489                  */
6490                 u32 switch_mode = hw->dev_caps.switch_mode &
6491                                   I40E_SWITCH_MODE_MASK;
6492                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6493                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6494                                 return 0;
6495                         dev_err(&pf->pdev->dev,
6496                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6497                                 hw->dev_caps.switch_mode);
6498                         return -EINVAL;
6499                 }
6500         }
6501
6502         /* Set Bit 7 to be valid */
6503         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6504
6505         /* Set L4type for TCP support */
6506         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6507
6508         /* Set cloud filter mode */
6509         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6510
6511         /* Prep mode field for set_switch_config */
6512         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6513                                         pf->last_sw_conf_valid_flags,
6514                                         mode, NULL);
6515         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6516                 dev_err(&pf->pdev->dev,
6517                         "couldn't set switch config bits, err %d aq_err %s\n",
6518                         ret,
6519                         i40e_aq_str(hw,
6520                                     hw->aq.asq_last_status));
6521
6522         return ret;
6523 }
6524
6525 /**
6526  * i40e_create_queue_channel - function to create channel
6527  * @vsi: VSI to be configured
6528  * @ch: ptr to channel (it contains channel specific params)
6529  *
6530  * This function creates channel (VSI) using num_queues specified by user,
6531  * reconfigs RSS if needed.
6532  **/
6533 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6534                               struct i40e_channel *ch)
6535 {
6536         struct i40e_pf *pf = vsi->back;
6537         bool reconfig_rss;
6538         int err;
6539
6540         if (!ch)
6541                 return -EINVAL;
6542
6543         if (!ch->num_queue_pairs) {
6544                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6545                         ch->num_queue_pairs);
6546                 return -EINVAL;
6547         }
6548
6549         /* validate user requested num_queues for channel */
6550         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6551                                        &reconfig_rss);
6552         if (err) {
6553                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6554                          ch->num_queue_pairs);
6555                 return -EINVAL;
6556         }
6557
6558         /* By default we are in VEPA mode, if this is the first VF/VMDq
6559          * VSI to be added switch to VEB mode.
6560          */
6561
6562         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6563                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6564
6565                 if (vsi->type == I40E_VSI_MAIN) {
6566                         if (i40e_is_tc_mqprio_enabled(pf))
6567                                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6568                         else
6569                                 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6570                 }
6571                 /* now onwards for main VSI, number of queues will be value
6572                  * of TC0's queue count
6573                  */
6574         }
6575
6576         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6577          * it should be more than num_queues
6578          */
6579         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6580                 dev_dbg(&pf->pdev->dev,
6581                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6582                         vsi->cnt_q_avail, ch->num_queue_pairs);
6583                 return -EINVAL;
6584         }
6585
6586         /* reconfig_rss only if vsi type is MAIN_VSI */
6587         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6588                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6589                 if (err) {
6590                         dev_info(&pf->pdev->dev,
6591                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6592                                  ch->num_queue_pairs);
6593                         return -EINVAL;
6594                 }
6595         }
6596
6597         if (!i40e_setup_channel(pf, vsi, ch)) {
6598                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6599                 return -EINVAL;
6600         }
6601
6602         dev_info(&pf->pdev->dev,
6603                  "Setup channel (id:%u) utilizing num_queues %d\n",
6604                  ch->seid, ch->num_queue_pairs);
6605
6606         /* configure VSI for BW limit */
6607         if (ch->max_tx_rate) {
6608                 u64 credits = ch->max_tx_rate;
6609
6610                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6611                         return -EINVAL;
6612
6613                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6614                 dev_dbg(&pf->pdev->dev,
6615                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6616                         ch->max_tx_rate,
6617                         credits,
6618                         ch->seid);
6619         }
6620
6621         /* in case of VF, this will be main SRIOV VSI */
6622         ch->parent_vsi = vsi;
6623
6624         /* and update main_vsi's count for queue_available to use */
6625         vsi->cnt_q_avail -= ch->num_queue_pairs;
6626
6627         return 0;
6628 }
6629
6630 /**
6631  * i40e_configure_queue_channels - Add queue channel for the given TCs
6632  * @vsi: VSI to be configured
6633  *
6634  * Configures queue channel mapping to the given TCs
6635  **/
6636 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6637 {
6638         struct i40e_channel *ch;
6639         u64 max_rate = 0;
6640         int ret = 0, i;
6641
6642         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6643         vsi->tc_seid_map[0] = vsi->seid;
6644         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6645                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6646                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6647                         if (!ch) {
6648                                 ret = -ENOMEM;
6649                                 goto err_free;
6650                         }
6651
6652                         INIT_LIST_HEAD(&ch->list);
6653                         ch->num_queue_pairs =
6654                                 vsi->tc_config.tc_info[i].qcount;
6655                         ch->base_queue =
6656                                 vsi->tc_config.tc_info[i].qoffset;
6657
6658                         /* Bandwidth limit through tc interface is in bytes/s,
6659                          * change to Mbit/s
6660                          */
6661                         max_rate = vsi->mqprio_qopt.max_rate[i];
6662                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6663                         ch->max_tx_rate = max_rate;
6664
6665                         list_add_tail(&ch->list, &vsi->ch_list);
6666
6667                         ret = i40e_create_queue_channel(vsi, ch);
6668                         if (ret) {
6669                                 dev_err(&vsi->back->pdev->dev,
6670                                         "Failed creating queue channel with TC%d: queues %d\n",
6671                                         i, ch->num_queue_pairs);
6672                                 goto err_free;
6673                         }
6674                         vsi->tc_seid_map[i] = ch->seid;
6675                 }
6676         }
6677
6678         /* reset to reconfigure TX queue contexts */
6679         i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6680         return ret;
6681
6682 err_free:
6683         i40e_remove_queue_channels(vsi);
6684         return ret;
6685 }
6686
6687 /**
6688  * i40e_veb_config_tc - Configure TCs for given VEB
6689  * @veb: given VEB
6690  * @enabled_tc: TC bitmap
6691  *
6692  * Configures given TC bitmap for VEB (switching) element
6693  **/
6694 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6695 {
6696         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6697         struct i40e_pf *pf = veb->pf;
6698         int ret = 0;
6699         int i;
6700
6701         /* No TCs or already enabled TCs just return */
6702         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6703                 return ret;
6704
6705         bw_data.tc_valid_bits = enabled_tc;
6706         /* bw_data.absolute_credits is not set (relative) */
6707
6708         /* Enable ETS TCs with equal BW Share for now */
6709         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6710                 if (enabled_tc & BIT(i))
6711                         bw_data.tc_bw_share_credits[i] = 1;
6712         }
6713
6714         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6715                                                    &bw_data, NULL);
6716         if (ret) {
6717                 dev_info(&pf->pdev->dev,
6718                          "VEB bw config failed, err %d aq_err %s\n",
6719                          ret,
6720                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6721                 goto out;
6722         }
6723
6724         /* Update the BW information */
6725         ret = i40e_veb_get_bw_info(veb);
6726         if (ret) {
6727                 dev_info(&pf->pdev->dev,
6728                          "Failed getting veb bw config, err %d aq_err %s\n",
6729                          ret,
6730                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6731         }
6732
6733 out:
6734         return ret;
6735 }
6736
6737 #ifdef CONFIG_I40E_DCB
6738 /**
6739  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6740  * @pf: PF struct
6741  *
6742  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6743  * the caller would've quiesce all the VSIs before calling
6744  * this function
6745  **/
6746 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6747 {
6748         u8 tc_map = 0;
6749         int ret;
6750         u8 v;
6751
6752         /* Enable the TCs available on PF to all VEBs */
6753         tc_map = i40e_pf_get_tc_map(pf);
6754         if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6755                 return;
6756
6757         for (v = 0; v < I40E_MAX_VEB; v++) {
6758                 if (!pf->veb[v])
6759                         continue;
6760                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6761                 if (ret) {
6762                         dev_info(&pf->pdev->dev,
6763                                  "Failed configuring TC for VEB seid=%d\n",
6764                                  pf->veb[v]->seid);
6765                         /* Will try to configure as many components */
6766                 }
6767         }
6768
6769         /* Update each VSI */
6770         for (v = 0; v < pf->num_alloc_vsi; v++) {
6771                 if (!pf->vsi[v])
6772                         continue;
6773
6774                 /* - Enable all TCs for the LAN VSI
6775                  * - For all others keep them at TC0 for now
6776                  */
6777                 if (v == pf->lan_vsi)
6778                         tc_map = i40e_pf_get_tc_map(pf);
6779                 else
6780                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6781
6782                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6783                 if (ret) {
6784                         dev_info(&pf->pdev->dev,
6785                                  "Failed configuring TC for VSI seid=%d\n",
6786                                  pf->vsi[v]->seid);
6787                         /* Will try to configure as many components */
6788                 } else {
6789                         /* Re-configure VSI vectors based on updated TC map */
6790                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6791                         if (pf->vsi[v]->netdev)
6792                                 i40e_dcbnl_set_all(pf->vsi[v]);
6793                 }
6794         }
6795 }
6796
6797 /**
6798  * i40e_resume_port_tx - Resume port Tx
6799  * @pf: PF struct
6800  *
6801  * Resume a port's Tx and issue a PF reset in case of failure to
6802  * resume.
6803  **/
6804 static int i40e_resume_port_tx(struct i40e_pf *pf)
6805 {
6806         struct i40e_hw *hw = &pf->hw;
6807         int ret;
6808
6809         ret = i40e_aq_resume_port_tx(hw, NULL);
6810         if (ret) {
6811                 dev_info(&pf->pdev->dev,
6812                          "Resume Port Tx failed, err %d aq_err %s\n",
6813                           ret,
6814                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6815                 /* Schedule PF reset to recover */
6816                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6817                 i40e_service_event_schedule(pf);
6818         }
6819
6820         return ret;
6821 }
6822
6823 /**
6824  * i40e_suspend_port_tx - Suspend port Tx
6825  * @pf: PF struct
6826  *
6827  * Suspend a port's Tx and issue a PF reset in case of failure.
6828  **/
6829 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6830 {
6831         struct i40e_hw *hw = &pf->hw;
6832         int ret;
6833
6834         ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6835         if (ret) {
6836                 dev_info(&pf->pdev->dev,
6837                          "Suspend Port Tx failed, err %d aq_err %s\n",
6838                          ret,
6839                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6840                 /* Schedule PF reset to recover */
6841                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6842                 i40e_service_event_schedule(pf);
6843         }
6844
6845         return ret;
6846 }
6847
6848 /**
6849  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6850  * @pf: PF being configured
6851  * @new_cfg: New DCBX configuration
6852  *
6853  * Program DCB settings into HW and reconfigure VEB/VSIs on
6854  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6855  **/
6856 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6857                                   struct i40e_dcbx_config *new_cfg)
6858 {
6859         struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6860         int ret;
6861
6862         /* Check if need reconfiguration */
6863         if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6864                 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6865                 return 0;
6866         }
6867
6868         /* Config change disable all VSIs */
6869         i40e_pf_quiesce_all_vsi(pf);
6870
6871         /* Copy the new config to the current config */
6872         *old_cfg = *new_cfg;
6873         old_cfg->etsrec = old_cfg->etscfg;
6874         ret = i40e_set_dcb_config(&pf->hw);
6875         if (ret) {
6876                 dev_info(&pf->pdev->dev,
6877                          "Set DCB Config failed, err %d aq_err %s\n",
6878                          ret,
6879                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6880                 goto out;
6881         }
6882
6883         /* Changes in configuration update VEB/VSI */
6884         i40e_dcb_reconfigure(pf);
6885 out:
6886         /* In case of reset do not try to resume anything */
6887         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6888                 /* Re-start the VSIs if disabled */
6889                 ret = i40e_resume_port_tx(pf);
6890                 /* In case of error no point in resuming VSIs */
6891                 if (ret)
6892                         goto err;
6893                 i40e_pf_unquiesce_all_vsi(pf);
6894         }
6895 err:
6896         return ret;
6897 }
6898
6899 /**
6900  * i40e_hw_dcb_config - Program new DCBX settings into HW
6901  * @pf: PF being configured
6902  * @new_cfg: New DCBX configuration
6903  *
6904  * Program DCB settings into HW and reconfigure VEB/VSIs on
6905  * given PF
6906  **/
6907 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6908 {
6909         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6910         u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6911         u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6912         struct i40e_dcbx_config *old_cfg;
6913         u8 mode[I40E_MAX_TRAFFIC_CLASS];
6914         struct i40e_rx_pb_config pb_cfg;
6915         struct i40e_hw *hw = &pf->hw;
6916         u8 num_ports = hw->num_ports;
6917         bool need_reconfig;
6918         int ret = -EINVAL;
6919         u8 lltc_map = 0;
6920         u8 tc_map = 0;
6921         u8 new_numtc;
6922         u8 i;
6923
6924         dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6925         /* Un-pack information to Program ETS HW via shared API
6926          * numtc, tcmap
6927          * LLTC map
6928          * ETS/NON-ETS arbiter mode
6929          * max exponent (credit refills)
6930          * Total number of ports
6931          * PFC priority bit-map
6932          * Priority Table
6933          * BW % per TC
6934          * Arbiter mode between UPs sharing same TC
6935          * TSA table (ETS or non-ETS)
6936          * EEE enabled or not
6937          * MFS TC table
6938          */
6939
6940         new_numtc = i40e_dcb_get_num_tc(new_cfg);
6941
6942         memset(&ets_data, 0, sizeof(ets_data));
6943         for (i = 0; i < new_numtc; i++) {
6944                 tc_map |= BIT(i);
6945                 switch (new_cfg->etscfg.tsatable[i]) {
6946                 case I40E_IEEE_TSA_ETS:
6947                         prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6948                         ets_data.tc_bw_share_credits[i] =
6949                                         new_cfg->etscfg.tcbwtable[i];
6950                         break;
6951                 case I40E_IEEE_TSA_STRICT:
6952                         prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6953                         lltc_map |= BIT(i);
6954                         ets_data.tc_bw_share_credits[i] =
6955                                         I40E_DCB_STRICT_PRIO_CREDITS;
6956                         break;
6957                 default:
6958                         /* Invalid TSA type */
6959                         need_reconfig = false;
6960                         goto out;
6961                 }
6962         }
6963
6964         old_cfg = &hw->local_dcbx_config;
6965         /* Check if need reconfiguration */
6966         need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
6967
6968         /* If needed, enable/disable frame tagging, disable all VSIs
6969          * and suspend port tx
6970          */
6971         if (need_reconfig) {
6972                 /* Enable DCB tagging only when more than one TC */
6973                 if (new_numtc > 1)
6974                         pf->flags |= I40E_FLAG_DCB_ENABLED;
6975                 else
6976                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6977
6978                 set_bit(__I40E_PORT_SUSPENDED, pf->state);
6979                 /* Reconfiguration needed quiesce all VSIs */
6980                 i40e_pf_quiesce_all_vsi(pf);
6981                 ret = i40e_suspend_port_tx(pf);
6982                 if (ret)
6983                         goto err;
6984         }
6985
6986         /* Configure Port ETS Tx Scheduler */
6987         ets_data.tc_valid_bits = tc_map;
6988         ets_data.tc_strict_priority_flags = lltc_map;
6989         ret = i40e_aq_config_switch_comp_ets
6990                 (hw, pf->mac_seid, &ets_data,
6991                  i40e_aqc_opc_modify_switching_comp_ets, NULL);
6992         if (ret) {
6993                 dev_info(&pf->pdev->dev,
6994                          "Modify Port ETS failed, err %d aq_err %s\n",
6995                          ret,
6996                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6997                 goto out;
6998         }
6999
7000         /* Configure Rx ETS HW */
7001         memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7002         i40e_dcb_hw_set_num_tc(hw, new_numtc);
7003         i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7004                                    I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7005                                    I40E_DCB_DEFAULT_MAX_EXPONENT,
7006                                    lltc_map);
7007         i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7008         i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7009                                      prio_type);
7010         i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7011                                new_cfg->etscfg.prioritytable);
7012         i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7013
7014         /* Configure Rx Packet Buffers in HW */
7015         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7016                 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
7017                 mfs_tc[i] += I40E_PACKET_HDR_PAD;
7018         }
7019
7020         i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7021                                          false, new_cfg->pfc.pfcenable,
7022                                          mfs_tc, &pb_cfg);
7023         i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7024
7025         /* Update the local Rx Packet buffer config */
7026         pf->pb_cfg = pb_cfg;
7027
7028         /* Inform the FW about changes to DCB configuration */
7029         ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7030         if (ret) {
7031                 dev_info(&pf->pdev->dev,
7032                          "DCB Updated failed, err %d aq_err %s\n",
7033                          ret,
7034                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7035                 goto out;
7036         }
7037
7038         /* Update the port DCBx configuration */
7039         *old_cfg = *new_cfg;
7040
7041         /* Changes in configuration update VEB/VSI */
7042         i40e_dcb_reconfigure(pf);
7043 out:
7044         /* Re-start the VSIs if disabled */
7045         if (need_reconfig) {
7046                 ret = i40e_resume_port_tx(pf);
7047
7048                 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7049                 /* In case of error no point in resuming VSIs */
7050                 if (ret)
7051                         goto err;
7052
7053                 /* Wait for the PF's queues to be disabled */
7054                 ret = i40e_pf_wait_queues_disabled(pf);
7055                 if (ret) {
7056                         /* Schedule PF reset to recover */
7057                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7058                         i40e_service_event_schedule(pf);
7059                         goto err;
7060                 } else {
7061                         i40e_pf_unquiesce_all_vsi(pf);
7062                         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7063                         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7064                 }
7065                 /* registers are set, lets apply */
7066                 if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
7067                         ret = i40e_hw_set_dcb_config(pf, new_cfg);
7068         }
7069
7070 err:
7071         return ret;
7072 }
7073
7074 /**
7075  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7076  * @pf: PF being queried
7077  *
7078  * Set default DCB configuration in case DCB is to be done in SW.
7079  **/
7080 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7081 {
7082         struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7083         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7084         struct i40e_hw *hw = &pf->hw;
7085         int err;
7086
7087         if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
7088                 /* Update the local cached instance with TC0 ETS */
7089                 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7090                 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7091                 pf->tmp_cfg.etscfg.maxtcs = 0;
7092                 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7093                 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7094                 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7095                 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7096                 /* FW needs one App to configure HW */
7097                 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7098                 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7099                 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7100                 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7101
7102                 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7103         }
7104
7105         memset(&ets_data, 0, sizeof(ets_data));
7106         ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7107         ets_data.tc_strict_priority_flags = 0; /* ETS */
7108         ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7109
7110         /* Enable ETS on the Physical port */
7111         err = i40e_aq_config_switch_comp_ets
7112                 (hw, pf->mac_seid, &ets_data,
7113                  i40e_aqc_opc_enable_switching_comp_ets, NULL);
7114         if (err) {
7115                 dev_info(&pf->pdev->dev,
7116                          "Enable Port ETS failed, err %d aq_err %s\n",
7117                          err,
7118                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7119                 err = -ENOENT;
7120                 goto out;
7121         }
7122
7123         /* Update the local cached instance with TC0 ETS */
7124         dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7125         dcb_cfg->etscfg.cbs = 0;
7126         dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7127         dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7128
7129 out:
7130         return err;
7131 }
7132
7133 /**
7134  * i40e_init_pf_dcb - Initialize DCB configuration
7135  * @pf: PF being configured
7136  *
7137  * Query the current DCB configuration and cache it
7138  * in the hardware structure
7139  **/
7140 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7141 {
7142         struct i40e_hw *hw = &pf->hw;
7143         int err;
7144
7145         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7146          * Also do not enable DCBx if FW LLDP agent is disabled
7147          */
7148         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
7149                 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7150                 err = I40E_NOT_SUPPORTED;
7151                 goto out;
7152         }
7153         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
7154                 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7155                 err = i40e_dcb_sw_default_config(pf);
7156                 if (err) {
7157                         dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7158                         goto out;
7159                 }
7160                 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7161                 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7162                                DCB_CAP_DCBX_VER_IEEE;
7163                 /* at init capable but disabled */
7164                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
7165                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7166                 goto out;
7167         }
7168         err = i40e_init_dcb(hw, true);
7169         if (!err) {
7170                 /* Device/Function is not DCBX capable */
7171                 if ((!hw->func_caps.dcb) ||
7172                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7173                         dev_info(&pf->pdev->dev,
7174                                  "DCBX offload is not supported or is disabled for this PF.\n");
7175                 } else {
7176                         /* When status is not DISABLED then DCBX in FW */
7177                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7178                                        DCB_CAP_DCBX_VER_IEEE;
7179
7180                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
7181                         /* Enable DCB tagging only when more than one TC
7182                          * or explicitly disable if only one TC
7183                          */
7184                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7185                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
7186                         else
7187                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7188                         dev_dbg(&pf->pdev->dev,
7189                                 "DCBX offload is supported for this PF.\n");
7190                 }
7191         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7192                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7193                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
7194         } else {
7195                 dev_info(&pf->pdev->dev,
7196                          "Query for DCB configuration failed, err %d aq_err %s\n",
7197                          err,
7198                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7199         }
7200
7201 out:
7202         return err;
7203 }
7204 #endif /* CONFIG_I40E_DCB */
7205
7206 /**
7207  * i40e_print_link_message - print link up or down
7208  * @vsi: the VSI for which link needs a message
7209  * @isup: true of link is up, false otherwise
7210  */
7211 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7212 {
7213         enum i40e_aq_link_speed new_speed;
7214         struct i40e_pf *pf = vsi->back;
7215         char *speed = "Unknown";
7216         char *fc = "Unknown";
7217         char *fec = "";
7218         char *req_fec = "";
7219         char *an = "";
7220
7221         if (isup)
7222                 new_speed = pf->hw.phy.link_info.link_speed;
7223         else
7224                 new_speed = I40E_LINK_SPEED_UNKNOWN;
7225
7226         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7227                 return;
7228         vsi->current_isup = isup;
7229         vsi->current_speed = new_speed;
7230         if (!isup) {
7231                 netdev_info(vsi->netdev, "NIC Link is Down\n");
7232                 return;
7233         }
7234
7235         /* Warn user if link speed on NPAR enabled partition is not at
7236          * least 10GB
7237          */
7238         if (pf->hw.func_caps.npar_enable &&
7239             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7240              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7241                 netdev_warn(vsi->netdev,
7242                             "The partition detected link speed that is less than 10Gbps\n");
7243
7244         switch (pf->hw.phy.link_info.link_speed) {
7245         case I40E_LINK_SPEED_40GB:
7246                 speed = "40 G";
7247                 break;
7248         case I40E_LINK_SPEED_20GB:
7249                 speed = "20 G";
7250                 break;
7251         case I40E_LINK_SPEED_25GB:
7252                 speed = "25 G";
7253                 break;
7254         case I40E_LINK_SPEED_10GB:
7255                 speed = "10 G";
7256                 break;
7257         case I40E_LINK_SPEED_5GB:
7258                 speed = "5 G";
7259                 break;
7260         case I40E_LINK_SPEED_2_5GB:
7261                 speed = "2.5 G";
7262                 break;
7263         case I40E_LINK_SPEED_1GB:
7264                 speed = "1000 M";
7265                 break;
7266         case I40E_LINK_SPEED_100MB:
7267                 speed = "100 M";
7268                 break;
7269         default:
7270                 break;
7271         }
7272
7273         switch (pf->hw.fc.current_mode) {
7274         case I40E_FC_FULL:
7275                 fc = "RX/TX";
7276                 break;
7277         case I40E_FC_TX_PAUSE:
7278                 fc = "TX";
7279                 break;
7280         case I40E_FC_RX_PAUSE:
7281                 fc = "RX";
7282                 break;
7283         default:
7284                 fc = "None";
7285                 break;
7286         }
7287
7288         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7289                 req_fec = "None";
7290                 fec = "None";
7291                 an = "False";
7292
7293                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7294                         an = "True";
7295
7296                 if (pf->hw.phy.link_info.fec_info &
7297                     I40E_AQ_CONFIG_FEC_KR_ENA)
7298                         fec = "CL74 FC-FEC/BASE-R";
7299                 else if (pf->hw.phy.link_info.fec_info &
7300                          I40E_AQ_CONFIG_FEC_RS_ENA)
7301                         fec = "CL108 RS-FEC";
7302
7303                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7304                  * both RS and FC are requested
7305                  */
7306                 if (vsi->back->hw.phy.link_info.req_fec_info &
7307                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7308                         if (vsi->back->hw.phy.link_info.req_fec_info &
7309                             I40E_AQ_REQUEST_FEC_RS)
7310                                 req_fec = "CL108 RS-FEC";
7311                         else
7312                                 req_fec = "CL74 FC-FEC/BASE-R";
7313                 }
7314                 netdev_info(vsi->netdev,
7315                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7316                             speed, req_fec, fec, an, fc);
7317         } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7318                 req_fec = "None";
7319                 fec = "None";
7320                 an = "False";
7321
7322                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7323                         an = "True";
7324
7325                 if (pf->hw.phy.link_info.fec_info &
7326                     I40E_AQ_CONFIG_FEC_KR_ENA)
7327                         fec = "CL74 FC-FEC/BASE-R";
7328
7329                 if (pf->hw.phy.link_info.req_fec_info &
7330                     I40E_AQ_REQUEST_FEC_KR)
7331                         req_fec = "CL74 FC-FEC/BASE-R";
7332
7333                 netdev_info(vsi->netdev,
7334                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7335                             speed, req_fec, fec, an, fc);
7336         } else {
7337                 netdev_info(vsi->netdev,
7338                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7339                             speed, fc);
7340         }
7341
7342 }
7343
7344 /**
7345  * i40e_up_complete - Finish the last steps of bringing up a connection
7346  * @vsi: the VSI being configured
7347  **/
7348 static int i40e_up_complete(struct i40e_vsi *vsi)
7349 {
7350         struct i40e_pf *pf = vsi->back;
7351         int err;
7352
7353         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7354                 i40e_vsi_configure_msix(vsi);
7355         else
7356                 i40e_configure_msi_and_legacy(vsi);
7357
7358         /* start rings */
7359         err = i40e_vsi_start_rings(vsi);
7360         if (err)
7361                 return err;
7362
7363         clear_bit(__I40E_VSI_DOWN, vsi->state);
7364         i40e_napi_enable_all(vsi);
7365         i40e_vsi_enable_irq(vsi);
7366
7367         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7368             (vsi->netdev)) {
7369                 i40e_print_link_message(vsi, true);
7370                 netif_tx_start_all_queues(vsi->netdev);
7371                 netif_carrier_on(vsi->netdev);
7372         }
7373
7374         /* replay FDIR SB filters */
7375         if (vsi->type == I40E_VSI_FDIR) {
7376                 /* reset fd counters */
7377                 pf->fd_add_err = 0;
7378                 pf->fd_atr_cnt = 0;
7379                 i40e_fdir_filter_restore(vsi);
7380         }
7381
7382         /* On the next run of the service_task, notify any clients of the new
7383          * opened netdev
7384          */
7385         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7386         i40e_service_event_schedule(pf);
7387
7388         return 0;
7389 }
7390
7391 /**
7392  * i40e_vsi_reinit_locked - Reset the VSI
7393  * @vsi: the VSI being configured
7394  *
7395  * Rebuild the ring structs after some configuration
7396  * has changed, e.g. MTU size.
7397  **/
7398 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7399 {
7400         struct i40e_pf *pf = vsi->back;
7401
7402         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7403                 usleep_range(1000, 2000);
7404         i40e_down(vsi);
7405
7406         i40e_up(vsi);
7407         clear_bit(__I40E_CONFIG_BUSY, pf->state);
7408 }
7409
7410 /**
7411  * i40e_force_link_state - Force the link status
7412  * @pf: board private structure
7413  * @is_up: whether the link state should be forced up or down
7414  **/
7415 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7416 {
7417         struct i40e_aq_get_phy_abilities_resp abilities;
7418         struct i40e_aq_set_phy_config config = {0};
7419         bool non_zero_phy_type = is_up;
7420         struct i40e_hw *hw = &pf->hw;
7421         u64 mask;
7422         u8 speed;
7423         int err;
7424
7425         /* Card might've been put in an unstable state by other drivers
7426          * and applications, which causes incorrect speed values being
7427          * set on startup. In order to clear speed registers, we call
7428          * get_phy_capabilities twice, once to get initial state of
7429          * available speeds, and once to get current PHY config.
7430          */
7431         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7432                                            NULL);
7433         if (err) {
7434                 dev_err(&pf->pdev->dev,
7435                         "failed to get phy cap., ret =  %d last_status =  %s\n",
7436                         err,
7437                         i40e_aq_str(hw, hw->aq.asq_last_status));
7438                 return err;
7439         }
7440         speed = abilities.link_speed;
7441
7442         /* Get the current phy config */
7443         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7444                                            NULL);
7445         if (err) {
7446                 dev_err(&pf->pdev->dev,
7447                         "failed to get phy cap., ret =  %d last_status =  %s\n",
7448                         err,
7449                         i40e_aq_str(hw, hw->aq.asq_last_status));
7450                 return err;
7451         }
7452
7453         /* If link needs to go up, but was not forced to go down,
7454          * and its speed values are OK, no need for a flap
7455          * if non_zero_phy_type was set, still need to force up
7456          */
7457         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
7458                 non_zero_phy_type = true;
7459         else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7460                 return I40E_SUCCESS;
7461
7462         /* To force link we need to set bits for all supported PHY types,
7463          * but there are now more than 32, so we need to split the bitmap
7464          * across two fields.
7465          */
7466         mask = I40E_PHY_TYPES_BITMASK;
7467         config.phy_type =
7468                 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7469         config.phy_type_ext =
7470                 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7471         /* Copy the old settings, except of phy_type */
7472         config.abilities = abilities.abilities;
7473         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
7474                 if (is_up)
7475                         config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7476                 else
7477                         config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7478         }
7479         if (abilities.link_speed != 0)
7480                 config.link_speed = abilities.link_speed;
7481         else
7482                 config.link_speed = speed;
7483         config.eee_capability = abilities.eee_capability;
7484         config.eeer = abilities.eeer_val;
7485         config.low_power_ctrl = abilities.d3_lpan;
7486         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7487                             I40E_AQ_PHY_FEC_CONFIG_MASK;
7488         err = i40e_aq_set_phy_config(hw, &config, NULL);
7489
7490         if (err) {
7491                 dev_err(&pf->pdev->dev,
7492                         "set phy config ret =  %d last_status =  %s\n",
7493                         err,
7494                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7495                 return err;
7496         }
7497
7498         /* Update the link info */
7499         err = i40e_update_link_info(hw);
7500         if (err) {
7501                 /* Wait a little bit (on 40G cards it sometimes takes a really
7502                  * long time for link to come back from the atomic reset)
7503                  * and try once more
7504                  */
7505                 msleep(1000);
7506                 i40e_update_link_info(hw);
7507         }
7508
7509         i40e_aq_set_link_restart_an(hw, is_up, NULL);
7510
7511         return I40E_SUCCESS;
7512 }
7513
7514 /**
7515  * i40e_up - Bring the connection back up after being down
7516  * @vsi: the VSI being configured
7517  **/
7518 int i40e_up(struct i40e_vsi *vsi)
7519 {
7520         int err;
7521
7522         if (vsi->type == I40E_VSI_MAIN &&
7523             (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7524              vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7525                 i40e_force_link_state(vsi->back, true);
7526
7527         err = i40e_vsi_configure(vsi);
7528         if (!err)
7529                 err = i40e_up_complete(vsi);
7530
7531         return err;
7532 }
7533
7534 /**
7535  * i40e_down - Shutdown the connection processing
7536  * @vsi: the VSI being stopped
7537  **/
7538 void i40e_down(struct i40e_vsi *vsi)
7539 {
7540         int i;
7541
7542         /* It is assumed that the caller of this function
7543          * sets the vsi->state __I40E_VSI_DOWN bit.
7544          */
7545         if (vsi->netdev) {
7546                 netif_carrier_off(vsi->netdev);
7547                 netif_tx_disable(vsi->netdev);
7548         }
7549         i40e_vsi_disable_irq(vsi);
7550         i40e_vsi_stop_rings(vsi);
7551         if (vsi->type == I40E_VSI_MAIN &&
7552            (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7553             vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7554                 i40e_force_link_state(vsi->back, false);
7555         i40e_napi_disable_all(vsi);
7556
7557         for (i = 0; i < vsi->num_queue_pairs; i++) {
7558                 i40e_clean_tx_ring(vsi->tx_rings[i]);
7559                 if (i40e_enabled_xdp_vsi(vsi)) {
7560                         /* Make sure that in-progress ndo_xdp_xmit and
7561                          * ndo_xsk_wakeup calls are completed.
7562                          */
7563                         synchronize_rcu();
7564                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
7565                 }
7566                 i40e_clean_rx_ring(vsi->rx_rings[i]);
7567         }
7568
7569 }
7570
7571 /**
7572  * i40e_validate_mqprio_qopt- validate queue mapping info
7573  * @vsi: the VSI being configured
7574  * @mqprio_qopt: queue parametrs
7575  **/
7576 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7577                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
7578 {
7579         u64 sum_max_rate = 0;
7580         u64 max_rate = 0;
7581         int i;
7582
7583         if (mqprio_qopt->qopt.offset[0] != 0 ||
7584             mqprio_qopt->qopt.num_tc < 1 ||
7585             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7586                 return -EINVAL;
7587         for (i = 0; ; i++) {
7588                 if (!mqprio_qopt->qopt.count[i])
7589                         return -EINVAL;
7590                 if (mqprio_qopt->min_rate[i]) {
7591                         dev_err(&vsi->back->pdev->dev,
7592                                 "Invalid min tx rate (greater than 0) specified\n");
7593                         return -EINVAL;
7594                 }
7595                 max_rate = mqprio_qopt->max_rate[i];
7596                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7597                 sum_max_rate += max_rate;
7598
7599                 if (i >= mqprio_qopt->qopt.num_tc - 1)
7600                         break;
7601                 if (mqprio_qopt->qopt.offset[i + 1] !=
7602                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7603                         return -EINVAL;
7604         }
7605         if (vsi->num_queue_pairs <
7606             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7607                 dev_err(&vsi->back->pdev->dev,
7608                         "Failed to create traffic channel, insufficient number of queues.\n");
7609                 return -EINVAL;
7610         }
7611         if (sum_max_rate > i40e_get_link_speed(vsi)) {
7612                 dev_err(&vsi->back->pdev->dev,
7613                         "Invalid max tx rate specified\n");
7614                 return -EINVAL;
7615         }
7616         return 0;
7617 }
7618
7619 /**
7620  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7621  * @vsi: the VSI being configured
7622  **/
7623 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7624 {
7625         u16 qcount;
7626         int i;
7627
7628         /* Only TC0 is enabled */
7629         vsi->tc_config.numtc = 1;
7630         vsi->tc_config.enabled_tc = 1;
7631         qcount = min_t(int, vsi->alloc_queue_pairs,
7632                        i40e_pf_get_max_q_per_tc(vsi->back));
7633         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7634                 /* For the TC that is not enabled set the offset to default
7635                  * queue and allocate one queue for the given TC.
7636                  */
7637                 vsi->tc_config.tc_info[i].qoffset = 0;
7638                 if (i == 0)
7639                         vsi->tc_config.tc_info[i].qcount = qcount;
7640                 else
7641                         vsi->tc_config.tc_info[i].qcount = 1;
7642                 vsi->tc_config.tc_info[i].netdev_tc = 0;
7643         }
7644 }
7645
7646 /**
7647  * i40e_del_macvlan_filter
7648  * @hw: pointer to the HW structure
7649  * @seid: seid of the channel VSI
7650  * @macaddr: the mac address to apply as a filter
7651  * @aq_err: store the admin Q error
7652  *
7653  * This function deletes a mac filter on the channel VSI which serves as the
7654  * macvlan. Returns 0 on success.
7655  **/
7656 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7657                                    const u8 *macaddr, int *aq_err)
7658 {
7659         struct i40e_aqc_remove_macvlan_element_data element;
7660         int status;
7661
7662         memset(&element, 0, sizeof(element));
7663         ether_addr_copy(element.mac_addr, macaddr);
7664         element.vlan_tag = 0;
7665         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7666         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7667         *aq_err = hw->aq.asq_last_status;
7668
7669         return status;
7670 }
7671
7672 /**
7673  * i40e_add_macvlan_filter
7674  * @hw: pointer to the HW structure
7675  * @seid: seid of the channel VSI
7676  * @macaddr: the mac address to apply as a filter
7677  * @aq_err: store the admin Q error
7678  *
7679  * This function adds a mac filter on the channel VSI which serves as the
7680  * macvlan. Returns 0 on success.
7681  **/
7682 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7683                                    const u8 *macaddr, int *aq_err)
7684 {
7685         struct i40e_aqc_add_macvlan_element_data element;
7686         u16 cmd_flags = 0;
7687         int status;
7688
7689         ether_addr_copy(element.mac_addr, macaddr);
7690         element.vlan_tag = 0;
7691         element.queue_number = 0;
7692         element.match_method = I40E_AQC_MM_ERR_NO_RES;
7693         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7694         element.flags = cpu_to_le16(cmd_flags);
7695         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7696         *aq_err = hw->aq.asq_last_status;
7697
7698         return status;
7699 }
7700
7701 /**
7702  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7703  * @vsi: the VSI we want to access
7704  * @ch: the channel we want to access
7705  */
7706 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7707 {
7708         struct i40e_ring *tx_ring, *rx_ring;
7709         u16 pf_q;
7710         int i;
7711
7712         for (i = 0; i < ch->num_queue_pairs; i++) {
7713                 pf_q = ch->base_queue + i;
7714                 tx_ring = vsi->tx_rings[pf_q];
7715                 tx_ring->ch = NULL;
7716                 rx_ring = vsi->rx_rings[pf_q];
7717                 rx_ring->ch = NULL;
7718         }
7719 }
7720
7721 /**
7722  * i40e_free_macvlan_channels
7723  * @vsi: the VSI we want to access
7724  *
7725  * This function frees the Qs of the channel VSI from
7726  * the stack and also deletes the channel VSIs which
7727  * serve as macvlans.
7728  */
7729 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7730 {
7731         struct i40e_channel *ch, *ch_tmp;
7732         int ret;
7733
7734         if (list_empty(&vsi->macvlan_list))
7735                 return;
7736
7737         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7738                 struct i40e_vsi *parent_vsi;
7739
7740                 if (i40e_is_channel_macvlan(ch)) {
7741                         i40e_reset_ch_rings(vsi, ch);
7742                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7743                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7744                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7745                         kfree(ch->fwd);
7746                         ch->fwd = NULL;
7747                 }
7748
7749                 list_del(&ch->list);
7750                 parent_vsi = ch->parent_vsi;
7751                 if (!parent_vsi || !ch->initialized) {
7752                         kfree(ch);
7753                         continue;
7754                 }
7755
7756                 /* remove the VSI */
7757                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7758                                              NULL);
7759                 if (ret)
7760                         dev_err(&vsi->back->pdev->dev,
7761                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7762                                 ch->seid, parent_vsi->seid);
7763                 kfree(ch);
7764         }
7765         vsi->macvlan_cnt = 0;
7766 }
7767
7768 /**
7769  * i40e_fwd_ring_up - bring the macvlan device up
7770  * @vsi: the VSI we want to access
7771  * @vdev: macvlan netdevice
7772  * @fwd: the private fwd structure
7773  */
7774 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7775                             struct i40e_fwd_adapter *fwd)
7776 {
7777         struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7778         int ret = 0, num_tc = 1,  i, aq_err;
7779         struct i40e_pf *pf = vsi->back;
7780         struct i40e_hw *hw = &pf->hw;
7781
7782         /* Go through the list and find an available channel */
7783         list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7784                 if (!i40e_is_channel_macvlan(iter)) {
7785                         iter->fwd = fwd;
7786                         /* record configuration for macvlan interface in vdev */
7787                         for (i = 0; i < num_tc; i++)
7788                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7789                                                              i,
7790                                                              iter->num_queue_pairs,
7791                                                              iter->base_queue);
7792                         for (i = 0; i < iter->num_queue_pairs; i++) {
7793                                 struct i40e_ring *tx_ring, *rx_ring;
7794                                 u16 pf_q;
7795
7796                                 pf_q = iter->base_queue + i;
7797
7798                                 /* Get to TX ring ptr */
7799                                 tx_ring = vsi->tx_rings[pf_q];
7800                                 tx_ring->ch = iter;
7801
7802                                 /* Get the RX ring ptr */
7803                                 rx_ring = vsi->rx_rings[pf_q];
7804                                 rx_ring->ch = iter;
7805                         }
7806                         ch = iter;
7807                         break;
7808                 }
7809         }
7810
7811         if (!ch)
7812                 return -EINVAL;
7813
7814         /* Guarantee all rings are updated before we update the
7815          * MAC address filter.
7816          */
7817         wmb();
7818
7819         /* Add a mac filter */
7820         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7821         if (ret) {
7822                 /* if we cannot add the MAC rule then disable the offload */
7823                 macvlan_release_l2fw_offload(vdev);
7824                 for (i = 0; i < ch->num_queue_pairs; i++) {
7825                         struct i40e_ring *rx_ring;
7826                         u16 pf_q;
7827
7828                         pf_q = ch->base_queue + i;
7829                         rx_ring = vsi->rx_rings[pf_q];
7830                         rx_ring->netdev = NULL;
7831                 }
7832                 dev_info(&pf->pdev->dev,
7833                          "Error adding mac filter on macvlan err %d, aq_err %s\n",
7834                           ret,
7835                           i40e_aq_str(hw, aq_err));
7836                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7837         }
7838
7839         return ret;
7840 }
7841
7842 /**
7843  * i40e_setup_macvlans - create the channels which will be macvlans
7844  * @vsi: the VSI we want to access
7845  * @macvlan_cnt: no. of macvlans to be setup
7846  * @qcnt: no. of Qs per macvlan
7847  * @vdev: macvlan netdevice
7848  */
7849 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7850                                struct net_device *vdev)
7851 {
7852         struct i40e_pf *pf = vsi->back;
7853         struct i40e_hw *hw = &pf->hw;
7854         struct i40e_vsi_context ctxt;
7855         u16 sections, qmap, num_qps;
7856         struct i40e_channel *ch;
7857         int i, pow, ret = 0;
7858         u8 offset = 0;
7859
7860         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7861                 return -EINVAL;
7862
7863         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7864
7865         /* find the next higher power-of-2 of num queue pairs */
7866         pow = fls(roundup_pow_of_two(num_qps) - 1);
7867
7868         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7869                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7870
7871         /* Setup context bits for the main VSI */
7872         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7873         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7874         memset(&ctxt, 0, sizeof(ctxt));
7875         ctxt.seid = vsi->seid;
7876         ctxt.pf_num = vsi->back->hw.pf_id;
7877         ctxt.vf_num = 0;
7878         ctxt.uplink_seid = vsi->uplink_seid;
7879         ctxt.info = vsi->info;
7880         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7881         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7882         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7883         ctxt.info.valid_sections |= cpu_to_le16(sections);
7884
7885         /* Reconfigure RSS for main VSI with new max queue count */
7886         vsi->rss_size = max_t(u16, num_qps, qcnt);
7887         ret = i40e_vsi_config_rss(vsi);
7888         if (ret) {
7889                 dev_info(&pf->pdev->dev,
7890                          "Failed to reconfig RSS for num_queues (%u)\n",
7891                          vsi->rss_size);
7892                 return ret;
7893         }
7894         vsi->reconfig_rss = true;
7895         dev_dbg(&vsi->back->pdev->dev,
7896                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7897         vsi->next_base_queue = num_qps;
7898         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7899
7900         /* Update the VSI after updating the VSI queue-mapping
7901          * information
7902          */
7903         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7904         if (ret) {
7905                 dev_info(&pf->pdev->dev,
7906                          "Update vsi tc config failed, err %d aq_err %s\n",
7907                          ret,
7908                          i40e_aq_str(hw, hw->aq.asq_last_status));
7909                 return ret;
7910         }
7911         /* update the local VSI info with updated queue map */
7912         i40e_vsi_update_queue_map(vsi, &ctxt);
7913         vsi->info.valid_sections = 0;
7914
7915         /* Create channels for macvlans */
7916         INIT_LIST_HEAD(&vsi->macvlan_list);
7917         for (i = 0; i < macvlan_cnt; i++) {
7918                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7919                 if (!ch) {
7920                         ret = -ENOMEM;
7921                         goto err_free;
7922                 }
7923                 INIT_LIST_HEAD(&ch->list);
7924                 ch->num_queue_pairs = qcnt;
7925                 if (!i40e_setup_channel(pf, vsi, ch)) {
7926                         ret = -EINVAL;
7927                         kfree(ch);
7928                         goto err_free;
7929                 }
7930                 ch->parent_vsi = vsi;
7931                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7932                 vsi->macvlan_cnt++;
7933                 list_add_tail(&ch->list, &vsi->macvlan_list);
7934         }
7935
7936         return ret;
7937
7938 err_free:
7939         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7940         i40e_free_macvlan_channels(vsi);
7941
7942         return ret;
7943 }
7944
7945 /**
7946  * i40e_fwd_add - configure macvlans
7947  * @netdev: net device to configure
7948  * @vdev: macvlan netdevice
7949  **/
7950 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7951 {
7952         struct i40e_netdev_priv *np = netdev_priv(netdev);
7953         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7954         struct i40e_vsi *vsi = np->vsi;
7955         struct i40e_pf *pf = vsi->back;
7956         struct i40e_fwd_adapter *fwd;
7957         int avail_macvlan, ret;
7958
7959         if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7960                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7961                 return ERR_PTR(-EINVAL);
7962         }
7963         if (i40e_is_tc_mqprio_enabled(pf)) {
7964                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7965                 return ERR_PTR(-EINVAL);
7966         }
7967         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7968                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7969                 return ERR_PTR(-EINVAL);
7970         }
7971
7972         /* The macvlan device has to be a single Q device so that the
7973          * tc_to_txq field can be reused to pick the tx queue.
7974          */
7975         if (netif_is_multiqueue(vdev))
7976                 return ERR_PTR(-ERANGE);
7977
7978         if (!vsi->macvlan_cnt) {
7979                 /* reserve bit 0 for the pf device */
7980                 set_bit(0, vsi->fwd_bitmask);
7981
7982                 /* Try to reserve as many queues as possible for macvlans. First
7983                  * reserve 3/4th of max vectors, then half, then quarter and
7984                  * calculate Qs per macvlan as you go
7985                  */
7986                 vectors = pf->num_lan_msix;
7987                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7988                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7989                         q_per_macvlan = 4;
7990                         macvlan_cnt = (vectors - 32) / 4;
7991                 } else if (vectors <= 64 && vectors > 32) {
7992                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7993                         q_per_macvlan = 2;
7994                         macvlan_cnt = (vectors - 16) / 2;
7995                 } else if (vectors <= 32 && vectors > 16) {
7996                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7997                         q_per_macvlan = 1;
7998                         macvlan_cnt = vectors - 16;
7999                 } else if (vectors <= 16 && vectors > 8) {
8000                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
8001                         q_per_macvlan = 1;
8002                         macvlan_cnt = vectors - 8;
8003                 } else {
8004                         /* allocate 1 Q per macvlan and 1 Q to the PF */
8005                         q_per_macvlan = 1;
8006                         macvlan_cnt = vectors - 1;
8007                 }
8008
8009                 if (macvlan_cnt == 0)
8010                         return ERR_PTR(-EBUSY);
8011
8012                 /* Quiesce VSI queues */
8013                 i40e_quiesce_vsi(vsi);
8014
8015                 /* sets up the macvlans but does not "enable" them */
8016                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8017                                           vdev);
8018                 if (ret)
8019                         return ERR_PTR(ret);
8020
8021                 /* Unquiesce VSI */
8022                 i40e_unquiesce_vsi(vsi);
8023         }
8024         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8025                                             vsi->macvlan_cnt);
8026         if (avail_macvlan >= I40E_MAX_MACVLANS)
8027                 return ERR_PTR(-EBUSY);
8028
8029         /* create the fwd struct */
8030         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8031         if (!fwd)
8032                 return ERR_PTR(-ENOMEM);
8033
8034         set_bit(avail_macvlan, vsi->fwd_bitmask);
8035         fwd->bit_no = avail_macvlan;
8036         netdev_set_sb_channel(vdev, avail_macvlan);
8037         fwd->netdev = vdev;
8038
8039         if (!netif_running(netdev))
8040                 return fwd;
8041
8042         /* Set fwd ring up */
8043         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8044         if (ret) {
8045                 /* unbind the queues and drop the subordinate channel config */
8046                 netdev_unbind_sb_channel(netdev, vdev);
8047                 netdev_set_sb_channel(vdev, 0);
8048
8049                 kfree(fwd);
8050                 return ERR_PTR(-EINVAL);
8051         }
8052
8053         return fwd;
8054 }
8055
8056 /**
8057  * i40e_del_all_macvlans - Delete all the mac filters on the channels
8058  * @vsi: the VSI we want to access
8059  */
8060 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8061 {
8062         struct i40e_channel *ch, *ch_tmp;
8063         struct i40e_pf *pf = vsi->back;
8064         struct i40e_hw *hw = &pf->hw;
8065         int aq_err, ret = 0;
8066
8067         if (list_empty(&vsi->macvlan_list))
8068                 return;
8069
8070         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8071                 if (i40e_is_channel_macvlan(ch)) {
8072                         ret = i40e_del_macvlan_filter(hw, ch->seid,
8073                                                       i40e_channel_mac(ch),
8074                                                       &aq_err);
8075                         if (!ret) {
8076                                 /* Reset queue contexts */
8077                                 i40e_reset_ch_rings(vsi, ch);
8078                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8079                                 netdev_unbind_sb_channel(vsi->netdev,
8080                                                          ch->fwd->netdev);
8081                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
8082                                 kfree(ch->fwd);
8083                                 ch->fwd = NULL;
8084                         }
8085                 }
8086         }
8087 }
8088
8089 /**
8090  * i40e_fwd_del - delete macvlan interfaces
8091  * @netdev: net device to configure
8092  * @vdev: macvlan netdevice
8093  */
8094 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8095 {
8096         struct i40e_netdev_priv *np = netdev_priv(netdev);
8097         struct i40e_fwd_adapter *fwd = vdev;
8098         struct i40e_channel *ch, *ch_tmp;
8099         struct i40e_vsi *vsi = np->vsi;
8100         struct i40e_pf *pf = vsi->back;
8101         struct i40e_hw *hw = &pf->hw;
8102         int aq_err, ret = 0;
8103
8104         /* Find the channel associated with the macvlan and del mac filter */
8105         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8106                 if (i40e_is_channel_macvlan(ch) &&
8107                     ether_addr_equal(i40e_channel_mac(ch),
8108                                      fwd->netdev->dev_addr)) {
8109                         ret = i40e_del_macvlan_filter(hw, ch->seid,
8110                                                       i40e_channel_mac(ch),
8111                                                       &aq_err);
8112                         if (!ret) {
8113                                 /* Reset queue contexts */
8114                                 i40e_reset_ch_rings(vsi, ch);
8115                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8116                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
8117                                 netdev_set_sb_channel(fwd->netdev, 0);
8118                                 kfree(ch->fwd);
8119                                 ch->fwd = NULL;
8120                         } else {
8121                                 dev_info(&pf->pdev->dev,
8122                                          "Error deleting mac filter on macvlan err %d, aq_err %s\n",
8123                                           ret,
8124                                           i40e_aq_str(hw, aq_err));
8125                         }
8126                         break;
8127                 }
8128         }
8129 }
8130
8131 /**
8132  * i40e_setup_tc - configure multiple traffic classes
8133  * @netdev: net device to configure
8134  * @type_data: tc offload data
8135  **/
8136 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8137 {
8138         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8139         struct i40e_netdev_priv *np = netdev_priv(netdev);
8140         struct i40e_vsi *vsi = np->vsi;
8141         struct i40e_pf *pf = vsi->back;
8142         u8 enabled_tc = 0, num_tc, hw;
8143         bool need_reset = false;
8144         int old_queue_pairs;
8145         int ret = -EINVAL;
8146         u16 mode;
8147         int i;
8148
8149         old_queue_pairs = vsi->num_queue_pairs;
8150         num_tc = mqprio_qopt->qopt.num_tc;
8151         hw = mqprio_qopt->qopt.hw;
8152         mode = mqprio_qopt->mode;
8153         if (!hw) {
8154                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
8155                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8156                 goto config_tc;
8157         }
8158
8159         /* Check if MFP enabled */
8160         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
8161                 netdev_info(netdev,
8162                             "Configuring TC not supported in MFP mode\n");
8163                 return ret;
8164         }
8165         switch (mode) {
8166         case TC_MQPRIO_MODE_DCB:
8167                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
8168
8169                 /* Check if DCB enabled to continue */
8170                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
8171                         netdev_info(netdev,
8172                                     "DCB is not enabled for adapter\n");
8173                         return ret;
8174                 }
8175
8176                 /* Check whether tc count is within enabled limit */
8177                 if (num_tc > i40e_pf_get_num_tc(pf)) {
8178                         netdev_info(netdev,
8179                                     "TC count greater than enabled on link for adapter\n");
8180                         return ret;
8181                 }
8182                 break;
8183         case TC_MQPRIO_MODE_CHANNEL:
8184                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
8185                         netdev_info(netdev,
8186                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8187                         return ret;
8188                 }
8189                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
8190                         return ret;
8191                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8192                 if (ret)
8193                         return ret;
8194                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8195                        sizeof(*mqprio_qopt));
8196                 pf->flags |= I40E_FLAG_TC_MQPRIO;
8197                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8198                 break;
8199         default:
8200                 return -EINVAL;
8201         }
8202
8203 config_tc:
8204         /* Generate TC map for number of tc requested */
8205         for (i = 0; i < num_tc; i++)
8206                 enabled_tc |= BIT(i);
8207
8208         /* Requesting same TC configuration as already enabled */
8209         if (enabled_tc == vsi->tc_config.enabled_tc &&
8210             mode != TC_MQPRIO_MODE_CHANNEL)
8211                 return 0;
8212
8213         /* Quiesce VSI queues */
8214         i40e_quiesce_vsi(vsi);
8215
8216         if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8217                 i40e_remove_queue_channels(vsi);
8218
8219         /* Configure VSI for enabled TCs */
8220         ret = i40e_vsi_config_tc(vsi, enabled_tc);
8221         if (ret) {
8222                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8223                             vsi->seid);
8224                 need_reset = true;
8225                 goto exit;
8226         } else if (enabled_tc &&
8227                    (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8228                 netdev_info(netdev,
8229                             "Failed to create channel. Override queues (%u) not power of 2\n",
8230                             vsi->tc_config.tc_info[0].qcount);
8231                 ret = -EINVAL;
8232                 need_reset = true;
8233                 goto exit;
8234         }
8235
8236         dev_info(&vsi->back->pdev->dev,
8237                  "Setup channel (id:%u) utilizing num_queues %d\n",
8238                  vsi->seid, vsi->tc_config.tc_info[0].qcount);
8239
8240         if (i40e_is_tc_mqprio_enabled(pf)) {
8241                 if (vsi->mqprio_qopt.max_rate[0]) {
8242                         u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8243                                                   vsi->mqprio_qopt.max_rate[0]);
8244
8245                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8246                         if (!ret) {
8247                                 u64 credits = max_tx_rate;
8248
8249                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
8250                                 dev_dbg(&vsi->back->pdev->dev,
8251                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8252                                         max_tx_rate,
8253                                         credits,
8254                                         vsi->seid);
8255                         } else {
8256                                 need_reset = true;
8257                                 goto exit;
8258                         }
8259                 }
8260                 ret = i40e_configure_queue_channels(vsi);
8261                 if (ret) {
8262                         vsi->num_queue_pairs = old_queue_pairs;
8263                         netdev_info(netdev,
8264                                     "Failed configuring queue channels\n");
8265                         need_reset = true;
8266                         goto exit;
8267                 }
8268         }
8269
8270 exit:
8271         /* Reset the configuration data to defaults, only TC0 is enabled */
8272         if (need_reset) {
8273                 i40e_vsi_set_default_tc_config(vsi);
8274                 need_reset = false;
8275         }
8276
8277         /* Unquiesce VSI */
8278         i40e_unquiesce_vsi(vsi);
8279         return ret;
8280 }
8281
8282 /**
8283  * i40e_set_cld_element - sets cloud filter element data
8284  * @filter: cloud filter rule
8285  * @cld: ptr to cloud filter element data
8286  *
8287  * This is helper function to copy data into cloud filter element
8288  **/
8289 static inline void
8290 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8291                      struct i40e_aqc_cloud_filters_element_data *cld)
8292 {
8293         u32 ipa;
8294         int i;
8295
8296         memset(cld, 0, sizeof(*cld));
8297         ether_addr_copy(cld->outer_mac, filter->dst_mac);
8298         ether_addr_copy(cld->inner_mac, filter->src_mac);
8299
8300         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8301                 return;
8302
8303         if (filter->n_proto == ETH_P_IPV6) {
8304 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
8305                 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8306                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8307
8308                         *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8309                 }
8310         } else {
8311                 ipa = be32_to_cpu(filter->dst_ipv4);
8312
8313                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8314         }
8315
8316         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8317
8318         /* tenant_id is not supported by FW now, once the support is enabled
8319          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8320          */
8321         if (filter->tenant_id)
8322                 return;
8323 }
8324
8325 /**
8326  * i40e_add_del_cloud_filter - Add/del cloud filter
8327  * @vsi: pointer to VSI
8328  * @filter: cloud filter rule
8329  * @add: if true, add, if false, delete
8330  *
8331  * Add or delete a cloud filter for a specific flow spec.
8332  * Returns 0 if the filter were successfully added.
8333  **/
8334 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8335                               struct i40e_cloud_filter *filter, bool add)
8336 {
8337         struct i40e_aqc_cloud_filters_element_data cld_filter;
8338         struct i40e_pf *pf = vsi->back;
8339         int ret;
8340         static const u16 flag_table[128] = {
8341                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8342                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8343                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8344                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8345                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8346                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8347                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8348                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8349                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8350                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8351                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8352                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8353                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8354                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
8355         };
8356
8357         if (filter->flags >= ARRAY_SIZE(flag_table))
8358                 return I40E_ERR_CONFIG;
8359
8360         memset(&cld_filter, 0, sizeof(cld_filter));
8361
8362         /* copy element needed to add cloud filter from filter */
8363         i40e_set_cld_element(filter, &cld_filter);
8364
8365         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8366                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8367                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8368
8369         if (filter->n_proto == ETH_P_IPV6)
8370                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8371                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8372         else
8373                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8374                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8375
8376         if (add)
8377                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8378                                                 &cld_filter, 1);
8379         else
8380                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8381                                                 &cld_filter, 1);
8382         if (ret)
8383                 dev_dbg(&pf->pdev->dev,
8384                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8385                         add ? "add" : "delete", filter->dst_port, ret,
8386                         pf->hw.aq.asq_last_status);
8387         else
8388                 dev_info(&pf->pdev->dev,
8389                          "%s cloud filter for VSI: %d\n",
8390                          add ? "Added" : "Deleted", filter->seid);
8391         return ret;
8392 }
8393
8394 /**
8395  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8396  * @vsi: pointer to VSI
8397  * @filter: cloud filter rule
8398  * @add: if true, add, if false, delete
8399  *
8400  * Add or delete a cloud filter for a specific flow spec using big buffer.
8401  * Returns 0 if the filter were successfully added.
8402  **/
8403 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8404                                       struct i40e_cloud_filter *filter,
8405                                       bool add)
8406 {
8407         struct i40e_aqc_cloud_filters_element_bb cld_filter;
8408         struct i40e_pf *pf = vsi->back;
8409         int ret;
8410
8411         /* Both (src/dst) valid mac_addr are not supported */
8412         if ((is_valid_ether_addr(filter->dst_mac) &&
8413              is_valid_ether_addr(filter->src_mac)) ||
8414             (is_multicast_ether_addr(filter->dst_mac) &&
8415              is_multicast_ether_addr(filter->src_mac)))
8416                 return -EOPNOTSUPP;
8417
8418         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8419          * ports are not supported via big buffer now.
8420          */
8421         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8422                 return -EOPNOTSUPP;
8423
8424         /* adding filter using src_port/src_ip is not supported at this stage */
8425         if (filter->src_port ||
8426             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8427             !ipv6_addr_any(&filter->ip.v6.src_ip6))
8428                 return -EOPNOTSUPP;
8429
8430         memset(&cld_filter, 0, sizeof(cld_filter));
8431
8432         /* copy element needed to add cloud filter from filter */
8433         i40e_set_cld_element(filter, &cld_filter.element);
8434
8435         if (is_valid_ether_addr(filter->dst_mac) ||
8436             is_valid_ether_addr(filter->src_mac) ||
8437             is_multicast_ether_addr(filter->dst_mac) ||
8438             is_multicast_ether_addr(filter->src_mac)) {
8439                 /* MAC + IP : unsupported mode */
8440                 if (filter->dst_ipv4)
8441                         return -EOPNOTSUPP;
8442
8443                 /* since we validated that L4 port must be valid before
8444                  * we get here, start with respective "flags" value
8445                  * and update if vlan is present or not
8446                  */
8447                 cld_filter.element.flags =
8448                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8449
8450                 if (filter->vlan_id) {
8451                         cld_filter.element.flags =
8452                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8453                 }
8454
8455         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8456                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8457                 cld_filter.element.flags =
8458                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8459                 if (filter->n_proto == ETH_P_IPV6)
8460                         cld_filter.element.flags |=
8461                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8462                 else
8463                         cld_filter.element.flags |=
8464                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8465         } else {
8466                 dev_err(&pf->pdev->dev,
8467                         "either mac or ip has to be valid for cloud filter\n");
8468                 return -EINVAL;
8469         }
8470
8471         /* Now copy L4 port in Byte 6..7 in general fields */
8472         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8473                                                 be16_to_cpu(filter->dst_port);
8474
8475         if (add) {
8476                 /* Validate current device switch mode, change if necessary */
8477                 ret = i40e_validate_and_set_switch_mode(vsi);
8478                 if (ret) {
8479                         dev_err(&pf->pdev->dev,
8480                                 "failed to set switch mode, ret %d\n",
8481                                 ret);
8482                         return ret;
8483                 }
8484
8485                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8486                                                    &cld_filter, 1);
8487         } else {
8488                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8489                                                    &cld_filter, 1);
8490         }
8491
8492         if (ret)
8493                 dev_dbg(&pf->pdev->dev,
8494                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8495                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8496         else
8497                 dev_info(&pf->pdev->dev,
8498                          "%s cloud filter for VSI: %d, L4 port: %d\n",
8499                          add ? "add" : "delete", filter->seid,
8500                          ntohs(filter->dst_port));
8501         return ret;
8502 }
8503
8504 /**
8505  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8506  * @vsi: Pointer to VSI
8507  * @f: Pointer to struct flow_cls_offload
8508  * @filter: Pointer to cloud filter structure
8509  *
8510  **/
8511 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8512                                  struct flow_cls_offload *f,
8513                                  struct i40e_cloud_filter *filter)
8514 {
8515         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8516         struct flow_dissector *dissector = rule->match.dissector;
8517         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8518         struct i40e_pf *pf = vsi->back;
8519         u8 field_flags = 0;
8520
8521         if (dissector->used_keys &
8522             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8523               BIT(FLOW_DISSECTOR_KEY_BASIC) |
8524               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8525               BIT(FLOW_DISSECTOR_KEY_VLAN) |
8526               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8527               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8528               BIT(FLOW_DISSECTOR_KEY_PORTS) |
8529               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8530                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8531                         dissector->used_keys);
8532                 return -EOPNOTSUPP;
8533         }
8534
8535         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8536                 struct flow_match_enc_keyid match;
8537
8538                 flow_rule_match_enc_keyid(rule, &match);
8539                 if (match.mask->keyid != 0)
8540                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8541
8542                 filter->tenant_id = be32_to_cpu(match.key->keyid);
8543         }
8544
8545         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8546                 struct flow_match_basic match;
8547
8548                 flow_rule_match_basic(rule, &match);
8549                 n_proto_key = ntohs(match.key->n_proto);
8550                 n_proto_mask = ntohs(match.mask->n_proto);
8551
8552                 if (n_proto_key == ETH_P_ALL) {
8553                         n_proto_key = 0;
8554                         n_proto_mask = 0;
8555                 }
8556                 filter->n_proto = n_proto_key & n_proto_mask;
8557                 filter->ip_proto = match.key->ip_proto;
8558         }
8559
8560         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8561                 struct flow_match_eth_addrs match;
8562
8563                 flow_rule_match_eth_addrs(rule, &match);
8564
8565                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8566                 if (!is_zero_ether_addr(match.mask->dst)) {
8567                         if (is_broadcast_ether_addr(match.mask->dst)) {
8568                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
8569                         } else {
8570                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8571                                         match.mask->dst);
8572                                 return I40E_ERR_CONFIG;
8573                         }
8574                 }
8575
8576                 if (!is_zero_ether_addr(match.mask->src)) {
8577                         if (is_broadcast_ether_addr(match.mask->src)) {
8578                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
8579                         } else {
8580                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8581                                         match.mask->src);
8582                                 return I40E_ERR_CONFIG;
8583                         }
8584                 }
8585                 ether_addr_copy(filter->dst_mac, match.key->dst);
8586                 ether_addr_copy(filter->src_mac, match.key->src);
8587         }
8588
8589         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8590                 struct flow_match_vlan match;
8591
8592                 flow_rule_match_vlan(rule, &match);
8593                 if (match.mask->vlan_id) {
8594                         if (match.mask->vlan_id == VLAN_VID_MASK) {
8595                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8596
8597                         } else {
8598                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8599                                         match.mask->vlan_id);
8600                                 return I40E_ERR_CONFIG;
8601                         }
8602                 }
8603
8604                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8605         }
8606
8607         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8608                 struct flow_match_control match;
8609
8610                 flow_rule_match_control(rule, &match);
8611                 addr_type = match.key->addr_type;
8612         }
8613
8614         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8615                 struct flow_match_ipv4_addrs match;
8616
8617                 flow_rule_match_ipv4_addrs(rule, &match);
8618                 if (match.mask->dst) {
8619                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8620                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8621                         } else {
8622                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8623                                         &match.mask->dst);
8624                                 return I40E_ERR_CONFIG;
8625                         }
8626                 }
8627
8628                 if (match.mask->src) {
8629                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
8630                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8631                         } else {
8632                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8633                                         &match.mask->src);
8634                                 return I40E_ERR_CONFIG;
8635                         }
8636                 }
8637
8638                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8639                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8640                         return I40E_ERR_CONFIG;
8641                 }
8642                 filter->dst_ipv4 = match.key->dst;
8643                 filter->src_ipv4 = match.key->src;
8644         }
8645
8646         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8647                 struct flow_match_ipv6_addrs match;
8648
8649                 flow_rule_match_ipv6_addrs(rule, &match);
8650
8651                 /* src and dest IPV6 address should not be LOOPBACK
8652                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
8653                  */
8654                 if (ipv6_addr_loopback(&match.key->dst) ||
8655                     ipv6_addr_loopback(&match.key->src)) {
8656                         dev_err(&pf->pdev->dev,
8657                                 "Bad ipv6, addr is LOOPBACK\n");
8658                         return I40E_ERR_CONFIG;
8659                 }
8660                 if (!ipv6_addr_any(&match.mask->dst) ||
8661                     !ipv6_addr_any(&match.mask->src))
8662                         field_flags |= I40E_CLOUD_FIELD_IIP;
8663
8664                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8665                        sizeof(filter->src_ipv6));
8666                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8667                        sizeof(filter->dst_ipv6));
8668         }
8669
8670         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8671                 struct flow_match_ports match;
8672
8673                 flow_rule_match_ports(rule, &match);
8674                 if (match.mask->src) {
8675                         if (match.mask->src == cpu_to_be16(0xffff)) {
8676                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8677                         } else {
8678                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8679                                         be16_to_cpu(match.mask->src));
8680                                 return I40E_ERR_CONFIG;
8681                         }
8682                 }
8683
8684                 if (match.mask->dst) {
8685                         if (match.mask->dst == cpu_to_be16(0xffff)) {
8686                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8687                         } else {
8688                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8689                                         be16_to_cpu(match.mask->dst));
8690                                 return I40E_ERR_CONFIG;
8691                         }
8692                 }
8693
8694                 filter->dst_port = match.key->dst;
8695                 filter->src_port = match.key->src;
8696
8697                 switch (filter->ip_proto) {
8698                 case IPPROTO_TCP:
8699                 case IPPROTO_UDP:
8700                         break;
8701                 default:
8702                         dev_err(&pf->pdev->dev,
8703                                 "Only UDP and TCP transport are supported\n");
8704                         return -EINVAL;
8705                 }
8706         }
8707         filter->flags = field_flags;
8708         return 0;
8709 }
8710
8711 /**
8712  * i40e_handle_tclass: Forward to a traffic class on the device
8713  * @vsi: Pointer to VSI
8714  * @tc: traffic class index on the device
8715  * @filter: Pointer to cloud filter structure
8716  *
8717  **/
8718 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8719                               struct i40e_cloud_filter *filter)
8720 {
8721         struct i40e_channel *ch, *ch_tmp;
8722
8723         /* direct to a traffic class on the same device */
8724         if (tc == 0) {
8725                 filter->seid = vsi->seid;
8726                 return 0;
8727         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8728                 if (!filter->dst_port) {
8729                         dev_err(&vsi->back->pdev->dev,
8730                                 "Specify destination port to direct to traffic class that is not default\n");
8731                         return -EINVAL;
8732                 }
8733                 if (list_empty(&vsi->ch_list))
8734                         return -EINVAL;
8735                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8736                                          list) {
8737                         if (ch->seid == vsi->tc_seid_map[tc])
8738                                 filter->seid = ch->seid;
8739                 }
8740                 return 0;
8741         }
8742         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8743         return -EINVAL;
8744 }
8745
8746 /**
8747  * i40e_configure_clsflower - Configure tc flower filters
8748  * @vsi: Pointer to VSI
8749  * @cls_flower: Pointer to struct flow_cls_offload
8750  *
8751  **/
8752 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8753                                     struct flow_cls_offload *cls_flower)
8754 {
8755         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8756         struct i40e_cloud_filter *filter = NULL;
8757         struct i40e_pf *pf = vsi->back;
8758         int err = 0;
8759
8760         if (tc < 0) {
8761                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8762                 return -EOPNOTSUPP;
8763         }
8764
8765         if (!tc) {
8766                 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8767                 return -EINVAL;
8768         }
8769
8770         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8771             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8772                 return -EBUSY;
8773
8774         if (pf->fdir_pf_active_filters ||
8775             (!hlist_empty(&pf->fdir_filter_list))) {
8776                 dev_err(&vsi->back->pdev->dev,
8777                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8778                 return -EINVAL;
8779         }
8780
8781         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8782                 dev_err(&vsi->back->pdev->dev,
8783                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8784                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8785                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8786         }
8787
8788         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8789         if (!filter)
8790                 return -ENOMEM;
8791
8792         filter->cookie = cls_flower->cookie;
8793
8794         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8795         if (err < 0)
8796                 goto err;
8797
8798         err = i40e_handle_tclass(vsi, tc, filter);
8799         if (err < 0)
8800                 goto err;
8801
8802         /* Add cloud filter */
8803         if (filter->dst_port)
8804                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8805         else
8806                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8807
8808         if (err) {
8809                 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8810                         err);
8811                 goto err;
8812         }
8813
8814         /* add filter to the ordered list */
8815         INIT_HLIST_NODE(&filter->cloud_node);
8816
8817         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8818
8819         pf->num_cloud_filters++;
8820
8821         return err;
8822 err:
8823         kfree(filter);
8824         return err;
8825 }
8826
8827 /**
8828  * i40e_find_cloud_filter - Find the could filter in the list
8829  * @vsi: Pointer to VSI
8830  * @cookie: filter specific cookie
8831  *
8832  **/
8833 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8834                                                         unsigned long *cookie)
8835 {
8836         struct i40e_cloud_filter *filter = NULL;
8837         struct hlist_node *node2;
8838
8839         hlist_for_each_entry_safe(filter, node2,
8840                                   &vsi->back->cloud_filter_list, cloud_node)
8841                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8842                         return filter;
8843         return NULL;
8844 }
8845
8846 /**
8847  * i40e_delete_clsflower - Remove tc flower filters
8848  * @vsi: Pointer to VSI
8849  * @cls_flower: Pointer to struct flow_cls_offload
8850  *
8851  **/
8852 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8853                                  struct flow_cls_offload *cls_flower)
8854 {
8855         struct i40e_cloud_filter *filter = NULL;
8856         struct i40e_pf *pf = vsi->back;
8857         int err = 0;
8858
8859         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8860
8861         if (!filter)
8862                 return -EINVAL;
8863
8864         hash_del(&filter->cloud_node);
8865
8866         if (filter->dst_port)
8867                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8868         else
8869                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8870
8871         kfree(filter);
8872         if (err) {
8873                 dev_err(&pf->pdev->dev,
8874                         "Failed to delete cloud filter, err %d\n", err);
8875                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8876         }
8877
8878         pf->num_cloud_filters--;
8879         if (!pf->num_cloud_filters)
8880                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8881                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8882                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8883                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8884                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8885                 }
8886         return 0;
8887 }
8888
8889 /**
8890  * i40e_setup_tc_cls_flower - flower classifier offloads
8891  * @np: net device to configure
8892  * @cls_flower: offload data
8893  **/
8894 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8895                                     struct flow_cls_offload *cls_flower)
8896 {
8897         struct i40e_vsi *vsi = np->vsi;
8898
8899         switch (cls_flower->command) {
8900         case FLOW_CLS_REPLACE:
8901                 return i40e_configure_clsflower(vsi, cls_flower);
8902         case FLOW_CLS_DESTROY:
8903                 return i40e_delete_clsflower(vsi, cls_flower);
8904         case FLOW_CLS_STATS:
8905                 return -EOPNOTSUPP;
8906         default:
8907                 return -EOPNOTSUPP;
8908         }
8909 }
8910
8911 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8912                                   void *cb_priv)
8913 {
8914         struct i40e_netdev_priv *np = cb_priv;
8915
8916         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8917                 return -EOPNOTSUPP;
8918
8919         switch (type) {
8920         case TC_SETUP_CLSFLOWER:
8921                 return i40e_setup_tc_cls_flower(np, type_data);
8922
8923         default:
8924                 return -EOPNOTSUPP;
8925         }
8926 }
8927
8928 static LIST_HEAD(i40e_block_cb_list);
8929
8930 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8931                            void *type_data)
8932 {
8933         struct i40e_netdev_priv *np = netdev_priv(netdev);
8934
8935         switch (type) {
8936         case TC_SETUP_QDISC_MQPRIO:
8937                 return i40e_setup_tc(netdev, type_data);
8938         case TC_SETUP_BLOCK:
8939                 return flow_block_cb_setup_simple(type_data,
8940                                                   &i40e_block_cb_list,
8941                                                   i40e_setup_tc_block_cb,
8942                                                   np, np, true);
8943         default:
8944                 return -EOPNOTSUPP;
8945         }
8946 }
8947
8948 /**
8949  * i40e_open - Called when a network interface is made active
8950  * @netdev: network interface device structure
8951  *
8952  * The open entry point is called when a network interface is made
8953  * active by the system (IFF_UP).  At this point all resources needed
8954  * for transmit and receive operations are allocated, the interrupt
8955  * handler is registered with the OS, the netdev watchdog subtask is
8956  * enabled, and the stack is notified that the interface is ready.
8957  *
8958  * Returns 0 on success, negative value on failure
8959  **/
8960 int i40e_open(struct net_device *netdev)
8961 {
8962         struct i40e_netdev_priv *np = netdev_priv(netdev);
8963         struct i40e_vsi *vsi = np->vsi;
8964         struct i40e_pf *pf = vsi->back;
8965         int err;
8966
8967         /* disallow open during test or if eeprom is broken */
8968         if (test_bit(__I40E_TESTING, pf->state) ||
8969             test_bit(__I40E_BAD_EEPROM, pf->state))
8970                 return -EBUSY;
8971
8972         netif_carrier_off(netdev);
8973
8974         if (i40e_force_link_state(pf, true))
8975                 return -EAGAIN;
8976
8977         err = i40e_vsi_open(vsi);
8978         if (err)
8979                 return err;
8980
8981         /* configure global TSO hardware offload settings */
8982         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8983                                                        TCP_FLAG_FIN) >> 16);
8984         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8985                                                        TCP_FLAG_FIN |
8986                                                        TCP_FLAG_CWR) >> 16);
8987         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8988         udp_tunnel_get_rx_info(netdev);
8989
8990         return 0;
8991 }
8992
8993 /**
8994  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
8995  * @vsi: vsi structure
8996  *
8997  * This updates netdev's number of tx/rx queues
8998  *
8999  * Returns status of setting tx/rx queues
9000  **/
9001 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9002 {
9003         int ret;
9004
9005         ret = netif_set_real_num_rx_queues(vsi->netdev,
9006                                            vsi->num_queue_pairs);
9007         if (ret)
9008                 return ret;
9009
9010         return netif_set_real_num_tx_queues(vsi->netdev,
9011                                             vsi->num_queue_pairs);
9012 }
9013
9014 /**
9015  * i40e_vsi_open -
9016  * @vsi: the VSI to open
9017  *
9018  * Finish initialization of the VSI.
9019  *
9020  * Returns 0 on success, negative value on failure
9021  *
9022  * Note: expects to be called while under rtnl_lock()
9023  **/
9024 int i40e_vsi_open(struct i40e_vsi *vsi)
9025 {
9026         struct i40e_pf *pf = vsi->back;
9027         char int_name[I40E_INT_NAME_STR_LEN];
9028         int err;
9029
9030         /* allocate descriptors */
9031         err = i40e_vsi_setup_tx_resources(vsi);
9032         if (err)
9033                 goto err_setup_tx;
9034         err = i40e_vsi_setup_rx_resources(vsi);
9035         if (err)
9036                 goto err_setup_rx;
9037
9038         err = i40e_vsi_configure(vsi);
9039         if (err)
9040                 goto err_setup_rx;
9041
9042         if (vsi->netdev) {
9043                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9044                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9045                 err = i40e_vsi_request_irq(vsi, int_name);
9046                 if (err)
9047                         goto err_setup_rx;
9048
9049                 /* Notify the stack of the actual queue counts. */
9050                 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9051                 if (err)
9052                         goto err_set_queues;
9053
9054         } else if (vsi->type == I40E_VSI_FDIR) {
9055                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9056                          dev_driver_string(&pf->pdev->dev),
9057                          dev_name(&pf->pdev->dev));
9058                 err = i40e_vsi_request_irq(vsi, int_name);
9059                 if (err)
9060                         goto err_setup_rx;
9061
9062         } else {
9063                 err = -EINVAL;
9064                 goto err_setup_rx;
9065         }
9066
9067         err = i40e_up_complete(vsi);
9068         if (err)
9069                 goto err_up_complete;
9070
9071         return 0;
9072
9073 err_up_complete:
9074         i40e_down(vsi);
9075 err_set_queues:
9076         i40e_vsi_free_irq(vsi);
9077 err_setup_rx:
9078         i40e_vsi_free_rx_resources(vsi);
9079 err_setup_tx:
9080         i40e_vsi_free_tx_resources(vsi);
9081         if (vsi == pf->vsi[pf->lan_vsi])
9082                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9083
9084         return err;
9085 }
9086
9087 /**
9088  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9089  * @pf: Pointer to PF
9090  *
9091  * This function destroys the hlist where all the Flow Director
9092  * filters were saved.
9093  **/
9094 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9095 {
9096         struct i40e_fdir_filter *filter;
9097         struct i40e_flex_pit *pit_entry, *tmp;
9098         struct hlist_node *node2;
9099
9100         hlist_for_each_entry_safe(filter, node2,
9101                                   &pf->fdir_filter_list, fdir_node) {
9102                 hlist_del(&filter->fdir_node);
9103                 kfree(filter);
9104         }
9105
9106         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9107                 list_del(&pit_entry->list);
9108                 kfree(pit_entry);
9109         }
9110         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9111
9112         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9113                 list_del(&pit_entry->list);
9114                 kfree(pit_entry);
9115         }
9116         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9117
9118         pf->fdir_pf_active_filters = 0;
9119         i40e_reset_fdir_filter_cnt(pf);
9120
9121         /* Reprogram the default input set for TCP/IPv4 */
9122         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9123                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9124                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9125
9126         /* Reprogram the default input set for TCP/IPv6 */
9127         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9128                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9129                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9130
9131         /* Reprogram the default input set for UDP/IPv4 */
9132         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9133                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9134                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9135
9136         /* Reprogram the default input set for UDP/IPv6 */
9137         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9138                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9139                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9140
9141         /* Reprogram the default input set for SCTP/IPv4 */
9142         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9143                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9144                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9145
9146         /* Reprogram the default input set for SCTP/IPv6 */
9147         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9148                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9149                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9150
9151         /* Reprogram the default input set for Other/IPv4 */
9152         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9153                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9154
9155         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9156                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9157
9158         /* Reprogram the default input set for Other/IPv6 */
9159         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9160                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9161
9162         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9163                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9164 }
9165
9166 /**
9167  * i40e_cloud_filter_exit - Cleans up the cloud filters
9168  * @pf: Pointer to PF
9169  *
9170  * This function destroys the hlist where all the cloud filters
9171  * were saved.
9172  **/
9173 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9174 {
9175         struct i40e_cloud_filter *cfilter;
9176         struct hlist_node *node;
9177
9178         hlist_for_each_entry_safe(cfilter, node,
9179                                   &pf->cloud_filter_list, cloud_node) {
9180                 hlist_del(&cfilter->cloud_node);
9181                 kfree(cfilter);
9182         }
9183         pf->num_cloud_filters = 0;
9184
9185         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
9186             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
9187                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9188                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
9189                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
9190         }
9191 }
9192
9193 /**
9194  * i40e_close - Disables a network interface
9195  * @netdev: network interface device structure
9196  *
9197  * The close entry point is called when an interface is de-activated
9198  * by the OS.  The hardware is still under the driver's control, but
9199  * this netdev interface is disabled.
9200  *
9201  * Returns 0, this is not allowed to fail
9202  **/
9203 int i40e_close(struct net_device *netdev)
9204 {
9205         struct i40e_netdev_priv *np = netdev_priv(netdev);
9206         struct i40e_vsi *vsi = np->vsi;
9207
9208         i40e_vsi_close(vsi);
9209
9210         return 0;
9211 }
9212
9213 /**
9214  * i40e_do_reset - Start a PF or Core Reset sequence
9215  * @pf: board private structure
9216  * @reset_flags: which reset is requested
9217  * @lock_acquired: indicates whether or not the lock has been acquired
9218  * before this function was called.
9219  *
9220  * The essential difference in resets is that the PF Reset
9221  * doesn't clear the packet buffers, doesn't reset the PE
9222  * firmware, and doesn't bother the other PFs on the chip.
9223  **/
9224 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9225 {
9226         u32 val;
9227
9228         /* do the biggest reset indicated */
9229         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9230
9231                 /* Request a Global Reset
9232                  *
9233                  * This will start the chip's countdown to the actual full
9234                  * chip reset event, and a warning interrupt to be sent
9235                  * to all PFs, including the requestor.  Our handler
9236                  * for the warning interrupt will deal with the shutdown
9237                  * and recovery of the switch setup.
9238                  */
9239                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9240                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9241                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9242                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9243
9244         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9245
9246                 /* Request a Core Reset
9247                  *
9248                  * Same as Global Reset, except does *not* include the MAC/PHY
9249                  */
9250                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9251                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9252                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
9253                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9254                 i40e_flush(&pf->hw);
9255
9256         } else if (reset_flags & I40E_PF_RESET_FLAG) {
9257
9258                 /* Request a PF Reset
9259                  *
9260                  * Resets only the PF-specific registers
9261                  *
9262                  * This goes directly to the tear-down and rebuild of
9263                  * the switch, since we need to do all the recovery as
9264                  * for the Core Reset.
9265                  */
9266                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
9267                 i40e_handle_reset_warning(pf, lock_acquired);
9268
9269         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9270                 /* Request a PF Reset
9271                  *
9272                  * Resets PF and reinitializes PFs VSI.
9273                  */
9274                 i40e_prep_for_reset(pf);
9275                 i40e_reset_and_rebuild(pf, true, lock_acquired);
9276                 dev_info(&pf->pdev->dev,
9277                          pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
9278                          "FW LLDP is disabled\n" :
9279                          "FW LLDP is enabled\n");
9280
9281         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9282                 int v;
9283
9284                 /* Find the VSI(s) that requested a re-init */
9285                 dev_info(&pf->pdev->dev,
9286                          "VSI reinit requested\n");
9287                 for (v = 0; v < pf->num_alloc_vsi; v++) {
9288                         struct i40e_vsi *vsi = pf->vsi[v];
9289
9290                         if (vsi != NULL &&
9291                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9292                                                vsi->state))
9293                                 i40e_vsi_reinit_locked(pf->vsi[v]);
9294                 }
9295         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9296                 int v;
9297
9298                 /* Find the VSI(s) that needs to be brought down */
9299                 dev_info(&pf->pdev->dev, "VSI down requested\n");
9300                 for (v = 0; v < pf->num_alloc_vsi; v++) {
9301                         struct i40e_vsi *vsi = pf->vsi[v];
9302
9303                         if (vsi != NULL &&
9304                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9305                                                vsi->state)) {
9306                                 set_bit(__I40E_VSI_DOWN, vsi->state);
9307                                 i40e_down(vsi);
9308                         }
9309                 }
9310         } else {
9311                 dev_info(&pf->pdev->dev,
9312                          "bad reset request 0x%08x\n", reset_flags);
9313         }
9314 }
9315
9316 #ifdef CONFIG_I40E_DCB
9317 /**
9318  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9319  * @pf: board private structure
9320  * @old_cfg: current DCB config
9321  * @new_cfg: new DCB config
9322  **/
9323 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9324                             struct i40e_dcbx_config *old_cfg,
9325                             struct i40e_dcbx_config *new_cfg)
9326 {
9327         bool need_reconfig = false;
9328
9329         /* Check if ETS configuration has changed */
9330         if (memcmp(&new_cfg->etscfg,
9331                    &old_cfg->etscfg,
9332                    sizeof(new_cfg->etscfg))) {
9333                 /* If Priority Table has changed reconfig is needed */
9334                 if (memcmp(&new_cfg->etscfg.prioritytable,
9335                            &old_cfg->etscfg.prioritytable,
9336                            sizeof(new_cfg->etscfg.prioritytable))) {
9337                         need_reconfig = true;
9338                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9339                 }
9340
9341                 if (memcmp(&new_cfg->etscfg.tcbwtable,
9342                            &old_cfg->etscfg.tcbwtable,
9343                            sizeof(new_cfg->etscfg.tcbwtable)))
9344                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9345
9346                 if (memcmp(&new_cfg->etscfg.tsatable,
9347                            &old_cfg->etscfg.tsatable,
9348                            sizeof(new_cfg->etscfg.tsatable)))
9349                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9350         }
9351
9352         /* Check if PFC configuration has changed */
9353         if (memcmp(&new_cfg->pfc,
9354                    &old_cfg->pfc,
9355                    sizeof(new_cfg->pfc))) {
9356                 need_reconfig = true;
9357                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9358         }
9359
9360         /* Check if APP Table has changed */
9361         if (memcmp(&new_cfg->app,
9362                    &old_cfg->app,
9363                    sizeof(new_cfg->app))) {
9364                 need_reconfig = true;
9365                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9366         }
9367
9368         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9369         return need_reconfig;
9370 }
9371
9372 /**
9373  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9374  * @pf: board private structure
9375  * @e: event info posted on ARQ
9376  **/
9377 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9378                                   struct i40e_arq_event_info *e)
9379 {
9380         struct i40e_aqc_lldp_get_mib *mib =
9381                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9382         struct i40e_hw *hw = &pf->hw;
9383         struct i40e_dcbx_config tmp_dcbx_cfg;
9384         bool need_reconfig = false;
9385         int ret = 0;
9386         u8 type;
9387
9388         /* X710-T*L 2.5G and 5G speeds don't support DCB */
9389         if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9390             (hw->phy.link_info.link_speed &
9391              ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9392              !(pf->flags & I40E_FLAG_DCB_CAPABLE))
9393                 /* let firmware decide if the DCB should be disabled */
9394                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
9395
9396         /* Not DCB capable or capability disabled */
9397         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
9398                 return ret;
9399
9400         /* Ignore if event is not for Nearest Bridge */
9401         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9402                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9403         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9404         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9405                 return ret;
9406
9407         /* Check MIB Type and return if event for Remote MIB update */
9408         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9409         dev_dbg(&pf->pdev->dev,
9410                 "LLDP event mib type %s\n", type ? "remote" : "local");
9411         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9412                 /* Update the remote cached instance and return */
9413                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9414                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9415                                 &hw->remote_dcbx_config);
9416                 goto exit;
9417         }
9418
9419         /* Store the old configuration */
9420         tmp_dcbx_cfg = hw->local_dcbx_config;
9421
9422         /* Reset the old DCBx configuration data */
9423         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9424         /* Get updated DCBX data from firmware */
9425         ret = i40e_get_dcb_config(&pf->hw);
9426         if (ret) {
9427                 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9428                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9429                     (hw->phy.link_info.link_speed &
9430                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9431                         dev_warn(&pf->pdev->dev,
9432                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9433                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9434                 } else {
9435                         dev_info(&pf->pdev->dev,
9436                                  "Failed querying DCB configuration data from firmware, err %d aq_err %s\n",
9437                                  ret,
9438                                  i40e_aq_str(&pf->hw,
9439                                              pf->hw.aq.asq_last_status));
9440                 }
9441                 goto exit;
9442         }
9443
9444         /* No change detected in DCBX configs */
9445         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9446                     sizeof(tmp_dcbx_cfg))) {
9447                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9448                 goto exit;
9449         }
9450
9451         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9452                                                &hw->local_dcbx_config);
9453
9454         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9455
9456         if (!need_reconfig)
9457                 goto exit;
9458
9459         /* Enable DCB tagging only when more than one TC */
9460         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9461                 pf->flags |= I40E_FLAG_DCB_ENABLED;
9462         else
9463                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9464
9465         set_bit(__I40E_PORT_SUSPENDED, pf->state);
9466         /* Reconfiguration needed quiesce all VSIs */
9467         i40e_pf_quiesce_all_vsi(pf);
9468
9469         /* Changes in configuration update VEB/VSI */
9470         i40e_dcb_reconfigure(pf);
9471
9472         ret = i40e_resume_port_tx(pf);
9473
9474         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9475         /* In case of error no point in resuming VSIs */
9476         if (ret)
9477                 goto exit;
9478
9479         /* Wait for the PF's queues to be disabled */
9480         ret = i40e_pf_wait_queues_disabled(pf);
9481         if (ret) {
9482                 /* Schedule PF reset to recover */
9483                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9484                 i40e_service_event_schedule(pf);
9485         } else {
9486                 i40e_pf_unquiesce_all_vsi(pf);
9487                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9488                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9489         }
9490
9491 exit:
9492         return ret;
9493 }
9494 #endif /* CONFIG_I40E_DCB */
9495
9496 /**
9497  * i40e_do_reset_safe - Protected reset path for userland calls.
9498  * @pf: board private structure
9499  * @reset_flags: which reset is requested
9500  *
9501  **/
9502 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9503 {
9504         rtnl_lock();
9505         i40e_do_reset(pf, reset_flags, true);
9506         rtnl_unlock();
9507 }
9508
9509 /**
9510  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9511  * @pf: board private structure
9512  * @e: event info posted on ARQ
9513  *
9514  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9515  * and VF queues
9516  **/
9517 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9518                                            struct i40e_arq_event_info *e)
9519 {
9520         struct i40e_aqc_lan_overflow *data =
9521                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9522         u32 queue = le32_to_cpu(data->prtdcb_rupto);
9523         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9524         struct i40e_hw *hw = &pf->hw;
9525         struct i40e_vf *vf;
9526         u16 vf_id;
9527
9528         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9529                 queue, qtx_ctl);
9530
9531         /* Queue belongs to VF, find the VF and issue VF reset */
9532         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9533             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9534                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9535                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9536                 vf_id -= hw->func_caps.vf_base_id;
9537                 vf = &pf->vf[vf_id];
9538                 i40e_vc_notify_vf_reset(vf);
9539                 /* Allow VF to process pending reset notification */
9540                 msleep(20);
9541                 i40e_reset_vf(vf, false);
9542         }
9543 }
9544
9545 /**
9546  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9547  * @pf: board private structure
9548  **/
9549 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9550 {
9551         u32 val, fcnt_prog;
9552
9553         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9554         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9555         return fcnt_prog;
9556 }
9557
9558 /**
9559  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9560  * @pf: board private structure
9561  **/
9562 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9563 {
9564         u32 val, fcnt_prog;
9565
9566         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9567         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9568                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9569                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9570         return fcnt_prog;
9571 }
9572
9573 /**
9574  * i40e_get_global_fd_count - Get total FD filters programmed on device
9575  * @pf: board private structure
9576  **/
9577 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9578 {
9579         u32 val, fcnt_prog;
9580
9581         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9582         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9583                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9584                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9585         return fcnt_prog;
9586 }
9587
9588 /**
9589  * i40e_reenable_fdir_sb - Restore FDir SB capability
9590  * @pf: board private structure
9591  **/
9592 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9593 {
9594         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9595                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9596                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9597                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9598 }
9599
9600 /**
9601  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9602  * @pf: board private structure
9603  **/
9604 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9605 {
9606         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9607                 /* ATR uses the same filtering logic as SB rules. It only
9608                  * functions properly if the input set mask is at the default
9609                  * settings. It is safe to restore the default input set
9610                  * because there are no active TCPv4 filter rules.
9611                  */
9612                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9613                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9614                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9615
9616                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9617                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9618                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9619         }
9620 }
9621
9622 /**
9623  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9624  * @pf: board private structure
9625  * @filter: FDir filter to remove
9626  */
9627 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9628                                        struct i40e_fdir_filter *filter)
9629 {
9630         /* Update counters */
9631         pf->fdir_pf_active_filters--;
9632         pf->fd_inv = 0;
9633
9634         switch (filter->flow_type) {
9635         case TCP_V4_FLOW:
9636                 pf->fd_tcp4_filter_cnt--;
9637                 break;
9638         case UDP_V4_FLOW:
9639                 pf->fd_udp4_filter_cnt--;
9640                 break;
9641         case SCTP_V4_FLOW:
9642                 pf->fd_sctp4_filter_cnt--;
9643                 break;
9644         case TCP_V6_FLOW:
9645                 pf->fd_tcp6_filter_cnt--;
9646                 break;
9647         case UDP_V6_FLOW:
9648                 pf->fd_udp6_filter_cnt--;
9649                 break;
9650         case SCTP_V6_FLOW:
9651                 pf->fd_udp6_filter_cnt--;
9652                 break;
9653         case IP_USER_FLOW:
9654                 switch (filter->ipl4_proto) {
9655                 case IPPROTO_TCP:
9656                         pf->fd_tcp4_filter_cnt--;
9657                         break;
9658                 case IPPROTO_UDP:
9659                         pf->fd_udp4_filter_cnt--;
9660                         break;
9661                 case IPPROTO_SCTP:
9662                         pf->fd_sctp4_filter_cnt--;
9663                         break;
9664                 case IPPROTO_IP:
9665                         pf->fd_ip4_filter_cnt--;
9666                         break;
9667                 }
9668                 break;
9669         case IPV6_USER_FLOW:
9670                 switch (filter->ipl4_proto) {
9671                 case IPPROTO_TCP:
9672                         pf->fd_tcp6_filter_cnt--;
9673                         break;
9674                 case IPPROTO_UDP:
9675                         pf->fd_udp6_filter_cnt--;
9676                         break;
9677                 case IPPROTO_SCTP:
9678                         pf->fd_sctp6_filter_cnt--;
9679                         break;
9680                 case IPPROTO_IP:
9681                         pf->fd_ip6_filter_cnt--;
9682                         break;
9683                 }
9684                 break;
9685         }
9686
9687         /* Remove the filter from the list and free memory */
9688         hlist_del(&filter->fdir_node);
9689         kfree(filter);
9690 }
9691
9692 /**
9693  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9694  * @pf: board private structure
9695  **/
9696 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9697 {
9698         struct i40e_fdir_filter *filter;
9699         u32 fcnt_prog, fcnt_avail;
9700         struct hlist_node *node;
9701
9702         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9703                 return;
9704
9705         /* Check if we have enough room to re-enable FDir SB capability. */
9706         fcnt_prog = i40e_get_global_fd_count(pf);
9707         fcnt_avail = pf->fdir_pf_filter_count;
9708         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9709             (pf->fd_add_err == 0) ||
9710             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9711                 i40e_reenable_fdir_sb(pf);
9712
9713         /* We should wait for even more space before re-enabling ATR.
9714          * Additionally, we cannot enable ATR as long as we still have TCP SB
9715          * rules active.
9716          */
9717         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9718             pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9719                 i40e_reenable_fdir_atr(pf);
9720
9721         /* if hw had a problem adding a filter, delete it */
9722         if (pf->fd_inv > 0) {
9723                 hlist_for_each_entry_safe(filter, node,
9724                                           &pf->fdir_filter_list, fdir_node)
9725                         if (filter->fd_id == pf->fd_inv)
9726                                 i40e_delete_invalid_filter(pf, filter);
9727         }
9728 }
9729
9730 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9731 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9732 /**
9733  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9734  * @pf: board private structure
9735  **/
9736 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9737 {
9738         unsigned long min_flush_time;
9739         int flush_wait_retry = 50;
9740         bool disable_atr = false;
9741         int fd_room;
9742         int reg;
9743
9744         if (!time_after(jiffies, pf->fd_flush_timestamp +
9745                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9746                 return;
9747
9748         /* If the flush is happening too quick and we have mostly SB rules we
9749          * should not re-enable ATR for some time.
9750          */
9751         min_flush_time = pf->fd_flush_timestamp +
9752                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9753         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9754
9755         if (!(time_after(jiffies, min_flush_time)) &&
9756             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9757                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9758                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9759                 disable_atr = true;
9760         }
9761
9762         pf->fd_flush_timestamp = jiffies;
9763         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9764         /* flush all filters */
9765         wr32(&pf->hw, I40E_PFQF_CTL_1,
9766              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9767         i40e_flush(&pf->hw);
9768         pf->fd_flush_cnt++;
9769         pf->fd_add_err = 0;
9770         do {
9771                 /* Check FD flush status every 5-6msec */
9772                 usleep_range(5000, 6000);
9773                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9774                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9775                         break;
9776         } while (flush_wait_retry--);
9777         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9778                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9779         } else {
9780                 /* replay sideband filters */
9781                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9782                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9783                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9784                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9785                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9786                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9787         }
9788 }
9789
9790 /**
9791  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9792  * @pf: board private structure
9793  **/
9794 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9795 {
9796         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9797 }
9798
9799 /**
9800  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9801  * @pf: board private structure
9802  **/
9803 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9804 {
9805
9806         /* if interface is down do nothing */
9807         if (test_bit(__I40E_DOWN, pf->state))
9808                 return;
9809
9810         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9811                 i40e_fdir_flush_and_replay(pf);
9812
9813         i40e_fdir_check_and_reenable(pf);
9814
9815 }
9816
9817 /**
9818  * i40e_vsi_link_event - notify VSI of a link event
9819  * @vsi: vsi to be notified
9820  * @link_up: link up or down
9821  **/
9822 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9823 {
9824         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9825                 return;
9826
9827         switch (vsi->type) {
9828         case I40E_VSI_MAIN:
9829                 if (!vsi->netdev || !vsi->netdev_registered)
9830                         break;
9831
9832                 if (link_up) {
9833                         netif_carrier_on(vsi->netdev);
9834                         netif_tx_wake_all_queues(vsi->netdev);
9835                 } else {
9836                         netif_carrier_off(vsi->netdev);
9837                         netif_tx_stop_all_queues(vsi->netdev);
9838                 }
9839                 break;
9840
9841         case I40E_VSI_SRIOV:
9842         case I40E_VSI_VMDQ2:
9843         case I40E_VSI_CTRL:
9844         case I40E_VSI_IWARP:
9845         case I40E_VSI_MIRROR:
9846         default:
9847                 /* there is no notification for other VSIs */
9848                 break;
9849         }
9850 }
9851
9852 /**
9853  * i40e_veb_link_event - notify elements on the veb of a link event
9854  * @veb: veb to be notified
9855  * @link_up: link up or down
9856  **/
9857 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9858 {
9859         struct i40e_pf *pf;
9860         int i;
9861
9862         if (!veb || !veb->pf)
9863                 return;
9864         pf = veb->pf;
9865
9866         /* depth first... */
9867         for (i = 0; i < I40E_MAX_VEB; i++)
9868                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9869                         i40e_veb_link_event(pf->veb[i], link_up);
9870
9871         /* ... now the local VSIs */
9872         for (i = 0; i < pf->num_alloc_vsi; i++)
9873                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9874                         i40e_vsi_link_event(pf->vsi[i], link_up);
9875 }
9876
9877 /**
9878  * i40e_link_event - Update netif_carrier status
9879  * @pf: board private structure
9880  **/
9881 static void i40e_link_event(struct i40e_pf *pf)
9882 {
9883         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9884         u8 new_link_speed, old_link_speed;
9885         bool new_link, old_link;
9886         int status;
9887 #ifdef CONFIG_I40E_DCB
9888         int err;
9889 #endif /* CONFIG_I40E_DCB */
9890
9891         /* set this to force the get_link_status call to refresh state */
9892         pf->hw.phy.get_link_info = true;
9893         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9894         status = i40e_get_link_status(&pf->hw, &new_link);
9895
9896         /* On success, disable temp link polling */
9897         if (status == I40E_SUCCESS) {
9898                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9899         } else {
9900                 /* Enable link polling temporarily until i40e_get_link_status
9901                  * returns I40E_SUCCESS
9902                  */
9903                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9904                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9905                         status);
9906                 return;
9907         }
9908
9909         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9910         new_link_speed = pf->hw.phy.link_info.link_speed;
9911
9912         if (new_link == old_link &&
9913             new_link_speed == old_link_speed &&
9914             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9915              new_link == netif_carrier_ok(vsi->netdev)))
9916                 return;
9917
9918         i40e_print_link_message(vsi, new_link);
9919
9920         /* Notify the base of the switch tree connected to
9921          * the link.  Floating VEBs are not notified.
9922          */
9923         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9924                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9925         else
9926                 i40e_vsi_link_event(vsi, new_link);
9927
9928         if (pf->vf)
9929                 i40e_vc_notify_link_state(pf);
9930
9931         if (pf->flags & I40E_FLAG_PTP)
9932                 i40e_ptp_set_increment(pf);
9933 #ifdef CONFIG_I40E_DCB
9934         if (new_link == old_link)
9935                 return;
9936         /* Not SW DCB so firmware will take care of default settings */
9937         if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9938                 return;
9939
9940         /* We cover here only link down, as after link up in case of SW DCB
9941          * SW LLDP agent will take care of setting it up
9942          */
9943         if (!new_link) {
9944                 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9945                 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9946                 err = i40e_dcb_sw_default_config(pf);
9947                 if (err) {
9948                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
9949                                        I40E_FLAG_DCB_ENABLED);
9950                 } else {
9951                         pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9952                                        DCB_CAP_DCBX_VER_IEEE;
9953                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
9954                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9955                 }
9956         }
9957 #endif /* CONFIG_I40E_DCB */
9958 }
9959
9960 /**
9961  * i40e_watchdog_subtask - periodic checks not using event driven response
9962  * @pf: board private structure
9963  **/
9964 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9965 {
9966         int i;
9967
9968         /* if interface is down do nothing */
9969         if (test_bit(__I40E_DOWN, pf->state) ||
9970             test_bit(__I40E_CONFIG_BUSY, pf->state))
9971                 return;
9972
9973         /* make sure we don't do these things too often */
9974         if (time_before(jiffies, (pf->service_timer_previous +
9975                                   pf->service_timer_period)))
9976                 return;
9977         pf->service_timer_previous = jiffies;
9978
9979         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9980             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9981                 i40e_link_event(pf);
9982
9983         /* Update the stats for active netdevs so the network stack
9984          * can look at updated numbers whenever it cares to
9985          */
9986         for (i = 0; i < pf->num_alloc_vsi; i++)
9987                 if (pf->vsi[i] && pf->vsi[i]->netdev)
9988                         i40e_update_stats(pf->vsi[i]);
9989
9990         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9991                 /* Update the stats for the active switching components */
9992                 for (i = 0; i < I40E_MAX_VEB; i++)
9993                         if (pf->veb[i])
9994                                 i40e_update_veb_stats(pf->veb[i]);
9995         }
9996
9997         i40e_ptp_rx_hang(pf);
9998         i40e_ptp_tx_hang(pf);
9999 }
10000
10001 /**
10002  * i40e_reset_subtask - Set up for resetting the device and driver
10003  * @pf: board private structure
10004  **/
10005 static void i40e_reset_subtask(struct i40e_pf *pf)
10006 {
10007         u32 reset_flags = 0;
10008
10009         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10010                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10011                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10012         }
10013         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10014                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10015                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10016         }
10017         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10018                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10019                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10020         }
10021         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10022                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10023                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10024         }
10025         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10026                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10027                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10028         }
10029
10030         /* If there's a recovery already waiting, it takes
10031          * precedence before starting a new reset sequence.
10032          */
10033         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10034                 i40e_prep_for_reset(pf);
10035                 i40e_reset(pf);
10036                 i40e_rebuild(pf, false, false);
10037         }
10038
10039         /* If we're already down or resetting, just bail */
10040         if (reset_flags &&
10041             !test_bit(__I40E_DOWN, pf->state) &&
10042             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10043                 i40e_do_reset(pf, reset_flags, false);
10044         }
10045 }
10046
10047 /**
10048  * i40e_handle_link_event - Handle link event
10049  * @pf: board private structure
10050  * @e: event info posted on ARQ
10051  **/
10052 static void i40e_handle_link_event(struct i40e_pf *pf,
10053                                    struct i40e_arq_event_info *e)
10054 {
10055         struct i40e_aqc_get_link_status *status =
10056                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10057
10058         /* Do a new status request to re-enable LSE reporting
10059          * and load new status information into the hw struct
10060          * This completely ignores any state information
10061          * in the ARQ event info, instead choosing to always
10062          * issue the AQ update link status command.
10063          */
10064         i40e_link_event(pf);
10065
10066         /* Check if module meets thermal requirements */
10067         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10068                 dev_err(&pf->pdev->dev,
10069                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10070                 dev_err(&pf->pdev->dev,
10071                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10072         } else {
10073                 /* check for unqualified module, if link is down, suppress
10074                  * the message if link was forced to be down.
10075                  */
10076                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10077                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10078                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
10079                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
10080                         dev_err(&pf->pdev->dev,
10081                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10082                         dev_err(&pf->pdev->dev,
10083                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10084                 }
10085         }
10086 }
10087
10088 /**
10089  * i40e_clean_adminq_subtask - Clean the AdminQ rings
10090  * @pf: board private structure
10091  **/
10092 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10093 {
10094         struct i40e_arq_event_info event;
10095         struct i40e_hw *hw = &pf->hw;
10096         u16 pending, i = 0;
10097         u16 opcode;
10098         u32 oldval;
10099         int ret;
10100         u32 val;
10101
10102         /* Do not run clean AQ when PF reset fails */
10103         if (test_bit(__I40E_RESET_FAILED, pf->state))
10104                 return;
10105
10106         /* check for error indications */
10107         val = rd32(&pf->hw, pf->hw.aq.arq.len);
10108         oldval = val;
10109         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10110                 if (hw->debug_mask & I40E_DEBUG_AQ)
10111                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10112                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10113         }
10114         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10115                 if (hw->debug_mask & I40E_DEBUG_AQ)
10116                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10117                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10118                 pf->arq_overflows++;
10119         }
10120         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10121                 if (hw->debug_mask & I40E_DEBUG_AQ)
10122                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10123                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10124         }
10125         if (oldval != val)
10126                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
10127
10128         val = rd32(&pf->hw, pf->hw.aq.asq.len);
10129         oldval = val;
10130         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10131                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10132                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10133                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10134         }
10135         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10136                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10137                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10138                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10139         }
10140         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10141                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10142                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10143                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10144         }
10145         if (oldval != val)
10146                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
10147
10148         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10149         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10150         if (!event.msg_buf)
10151                 return;
10152
10153         do {
10154                 ret = i40e_clean_arq_element(hw, &event, &pending);
10155                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
10156                         break;
10157                 else if (ret) {
10158                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10159                         break;
10160                 }
10161
10162                 opcode = le16_to_cpu(event.desc.opcode);
10163                 switch (opcode) {
10164
10165                 case i40e_aqc_opc_get_link_status:
10166                         rtnl_lock();
10167                         i40e_handle_link_event(pf, &event);
10168                         rtnl_unlock();
10169                         break;
10170                 case i40e_aqc_opc_send_msg_to_pf:
10171                         ret = i40e_vc_process_vf_msg(pf,
10172                                         le16_to_cpu(event.desc.retval),
10173                                         le32_to_cpu(event.desc.cookie_high),
10174                                         le32_to_cpu(event.desc.cookie_low),
10175                                         event.msg_buf,
10176                                         event.msg_len);
10177                         break;
10178                 case i40e_aqc_opc_lldp_update_mib:
10179                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10180 #ifdef CONFIG_I40E_DCB
10181                         rtnl_lock();
10182                         i40e_handle_lldp_event(pf, &event);
10183                         rtnl_unlock();
10184 #endif /* CONFIG_I40E_DCB */
10185                         break;
10186                 case i40e_aqc_opc_event_lan_overflow:
10187                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10188                         i40e_handle_lan_overflow_event(pf, &event);
10189                         break;
10190                 case i40e_aqc_opc_send_msg_to_peer:
10191                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10192                         break;
10193                 case i40e_aqc_opc_nvm_erase:
10194                 case i40e_aqc_opc_nvm_update:
10195                 case i40e_aqc_opc_oem_post_update:
10196                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10197                                    "ARQ NVM operation 0x%04x completed\n",
10198                                    opcode);
10199                         break;
10200                 default:
10201                         dev_info(&pf->pdev->dev,
10202                                  "ARQ: Unknown event 0x%04x ignored\n",
10203                                  opcode);
10204                         break;
10205                 }
10206         } while (i++ < pf->adminq_work_limit);
10207
10208         if (i < pf->adminq_work_limit)
10209                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10210
10211         /* re-enable Admin queue interrupt cause */
10212         val = rd32(hw, I40E_PFINT_ICR0_ENA);
10213         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10214         wr32(hw, I40E_PFINT_ICR0_ENA, val);
10215         i40e_flush(hw);
10216
10217         kfree(event.msg_buf);
10218 }
10219
10220 /**
10221  * i40e_verify_eeprom - make sure eeprom is good to use
10222  * @pf: board private structure
10223  **/
10224 static void i40e_verify_eeprom(struct i40e_pf *pf)
10225 {
10226         int err;
10227
10228         err = i40e_diag_eeprom_test(&pf->hw);
10229         if (err) {
10230                 /* retry in case of garbage read */
10231                 err = i40e_diag_eeprom_test(&pf->hw);
10232                 if (err) {
10233                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10234                                  err);
10235                         set_bit(__I40E_BAD_EEPROM, pf->state);
10236                 }
10237         }
10238
10239         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10240                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10241                 clear_bit(__I40E_BAD_EEPROM, pf->state);
10242         }
10243 }
10244
10245 /**
10246  * i40e_enable_pf_switch_lb
10247  * @pf: pointer to the PF structure
10248  *
10249  * enable switch loop back or die - no point in a return value
10250  **/
10251 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10252 {
10253         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10254         struct i40e_vsi_context ctxt;
10255         int ret;
10256
10257         ctxt.seid = pf->main_vsi_seid;
10258         ctxt.pf_num = pf->hw.pf_id;
10259         ctxt.vf_num = 0;
10260         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10261         if (ret) {
10262                 dev_info(&pf->pdev->dev,
10263                          "couldn't get PF vsi config, err %d aq_err %s\n",
10264                          ret,
10265                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10266                 return;
10267         }
10268         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10269         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10270         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10271
10272         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10273         if (ret) {
10274                 dev_info(&pf->pdev->dev,
10275                          "update vsi switch failed, err %d aq_err %s\n",
10276                          ret,
10277                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10278         }
10279 }
10280
10281 /**
10282  * i40e_disable_pf_switch_lb
10283  * @pf: pointer to the PF structure
10284  *
10285  * disable switch loop back or die - no point in a return value
10286  **/
10287 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10288 {
10289         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10290         struct i40e_vsi_context ctxt;
10291         int ret;
10292
10293         ctxt.seid = pf->main_vsi_seid;
10294         ctxt.pf_num = pf->hw.pf_id;
10295         ctxt.vf_num = 0;
10296         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10297         if (ret) {
10298                 dev_info(&pf->pdev->dev,
10299                          "couldn't get PF vsi config, err %d aq_err %s\n",
10300                          ret,
10301                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10302                 return;
10303         }
10304         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10305         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10306         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10307
10308         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10309         if (ret) {
10310                 dev_info(&pf->pdev->dev,
10311                          "update vsi switch failed, err %d aq_err %s\n",
10312                          ret,
10313                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10314         }
10315 }
10316
10317 /**
10318  * i40e_config_bridge_mode - Configure the HW bridge mode
10319  * @veb: pointer to the bridge instance
10320  *
10321  * Configure the loop back mode for the LAN VSI that is downlink to the
10322  * specified HW bridge instance. It is expected this function is called
10323  * when a new HW bridge is instantiated.
10324  **/
10325 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10326 {
10327         struct i40e_pf *pf = veb->pf;
10328
10329         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10330                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10331                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10332         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10333                 i40e_disable_pf_switch_lb(pf);
10334         else
10335                 i40e_enable_pf_switch_lb(pf);
10336 }
10337
10338 /**
10339  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
10340  * @veb: pointer to the VEB instance
10341  *
10342  * This is a recursive function that first builds the attached VSIs then
10343  * recurses in to build the next layer of VEB.  We track the connections
10344  * through our own index numbers because the seid's from the HW could
10345  * change across the reset.
10346  **/
10347 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10348 {
10349         struct i40e_vsi *ctl_vsi = NULL;
10350         struct i40e_pf *pf = veb->pf;
10351         int v, veb_idx;
10352         int ret;
10353
10354         /* build VSI that owns this VEB, temporarily attached to base VEB */
10355         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
10356                 if (pf->vsi[v] &&
10357                     pf->vsi[v]->veb_idx == veb->idx &&
10358                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
10359                         ctl_vsi = pf->vsi[v];
10360                         break;
10361                 }
10362         }
10363         if (!ctl_vsi) {
10364                 dev_info(&pf->pdev->dev,
10365                          "missing owner VSI for veb_idx %d\n", veb->idx);
10366                 ret = -ENOENT;
10367                 goto end_reconstitute;
10368         }
10369         if (ctl_vsi != pf->vsi[pf->lan_vsi])
10370                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10371         ret = i40e_add_vsi(ctl_vsi);
10372         if (ret) {
10373                 dev_info(&pf->pdev->dev,
10374                          "rebuild of veb_idx %d owner VSI failed: %d\n",
10375                          veb->idx, ret);
10376                 goto end_reconstitute;
10377         }
10378         i40e_vsi_reset_stats(ctl_vsi);
10379
10380         /* create the VEB in the switch and move the VSI onto the VEB */
10381         ret = i40e_add_veb(veb, ctl_vsi);
10382         if (ret)
10383                 goto end_reconstitute;
10384
10385         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10386                 veb->bridge_mode = BRIDGE_MODE_VEB;
10387         else
10388                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10389         i40e_config_bridge_mode(veb);
10390
10391         /* create the remaining VSIs attached to this VEB */
10392         for (v = 0; v < pf->num_alloc_vsi; v++) {
10393                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
10394                         continue;
10395
10396                 if (pf->vsi[v]->veb_idx == veb->idx) {
10397                         struct i40e_vsi *vsi = pf->vsi[v];
10398
10399                         vsi->uplink_seid = veb->seid;
10400                         ret = i40e_add_vsi(vsi);
10401                         if (ret) {
10402                                 dev_info(&pf->pdev->dev,
10403                                          "rebuild of vsi_idx %d failed: %d\n",
10404                                          v, ret);
10405                                 goto end_reconstitute;
10406                         }
10407                         i40e_vsi_reset_stats(vsi);
10408                 }
10409         }
10410
10411         /* create any VEBs attached to this VEB - RECURSION */
10412         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10413                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
10414                         pf->veb[veb_idx]->uplink_seid = veb->seid;
10415                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
10416                         if (ret)
10417                                 break;
10418                 }
10419         }
10420
10421 end_reconstitute:
10422         return ret;
10423 }
10424
10425 /**
10426  * i40e_get_capabilities - get info about the HW
10427  * @pf: the PF struct
10428  * @list_type: AQ capability to be queried
10429  **/
10430 static int i40e_get_capabilities(struct i40e_pf *pf,
10431                                  enum i40e_admin_queue_opc list_type)
10432 {
10433         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10434         u16 data_size;
10435         int buf_len;
10436         int err;
10437
10438         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10439         do {
10440                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10441                 if (!cap_buf)
10442                         return -ENOMEM;
10443
10444                 /* this loads the data into the hw struct for us */
10445                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10446                                                     &data_size, list_type,
10447                                                     NULL);
10448                 /* data loaded, buffer no longer needed */
10449                 kfree(cap_buf);
10450
10451                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10452                         /* retry with a larger buffer */
10453                         buf_len = data_size;
10454                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10455                         dev_info(&pf->pdev->dev,
10456                                  "capability discovery failed, err %d aq_err %s\n",
10457                                  err,
10458                                  i40e_aq_str(&pf->hw,
10459                                              pf->hw.aq.asq_last_status));
10460                         return -ENODEV;
10461                 }
10462         } while (err);
10463
10464         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10465                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10466                         dev_info(&pf->pdev->dev,
10467                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10468                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10469                                  pf->hw.func_caps.num_msix_vectors,
10470                                  pf->hw.func_caps.num_msix_vectors_vf,
10471                                  pf->hw.func_caps.fd_filters_guaranteed,
10472                                  pf->hw.func_caps.fd_filters_best_effort,
10473                                  pf->hw.func_caps.num_tx_qp,
10474                                  pf->hw.func_caps.num_vsis);
10475                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10476                         dev_info(&pf->pdev->dev,
10477                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
10478                                  pf->hw.dev_caps.switch_mode,
10479                                  pf->hw.dev_caps.valid_functions);
10480                         dev_info(&pf->pdev->dev,
10481                                  "SR-IOV=%d, num_vfs for all function=%u\n",
10482                                  pf->hw.dev_caps.sr_iov_1_1,
10483                                  pf->hw.dev_caps.num_vfs);
10484                         dev_info(&pf->pdev->dev,
10485                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10486                                  pf->hw.dev_caps.num_vsis,
10487                                  pf->hw.dev_caps.num_rx_qp,
10488                                  pf->hw.dev_caps.num_tx_qp);
10489                 }
10490         }
10491         if (list_type == i40e_aqc_opc_list_func_capabilities) {
10492 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10493                        + pf->hw.func_caps.num_vfs)
10494                 if (pf->hw.revision_id == 0 &&
10495                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10496                         dev_info(&pf->pdev->dev,
10497                                  "got num_vsis %d, setting num_vsis to %d\n",
10498                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10499                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10500                 }
10501         }
10502         return 0;
10503 }
10504
10505 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10506
10507 /**
10508  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10509  * @pf: board private structure
10510  **/
10511 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10512 {
10513         struct i40e_vsi *vsi;
10514
10515         /* quick workaround for an NVM issue that leaves a critical register
10516          * uninitialized
10517          */
10518         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10519                 static const u32 hkey[] = {
10520                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10521                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10522                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10523                         0x95b3a76d};
10524                 int i;
10525
10526                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10527                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10528         }
10529
10530         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
10531                 return;
10532
10533         /* find existing VSI and see if it needs configuring */
10534         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10535
10536         /* create a new VSI if none exists */
10537         if (!vsi) {
10538                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10539                                      pf->vsi[pf->lan_vsi]->seid, 0);
10540                 if (!vsi) {
10541                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10542                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10543                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10544                         return;
10545                 }
10546         }
10547
10548         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10549 }
10550
10551 /**
10552  * i40e_fdir_teardown - release the Flow Director resources
10553  * @pf: board private structure
10554  **/
10555 static void i40e_fdir_teardown(struct i40e_pf *pf)
10556 {
10557         struct i40e_vsi *vsi;
10558
10559         i40e_fdir_filter_exit(pf);
10560         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10561         if (vsi)
10562                 i40e_vsi_release(vsi);
10563 }
10564
10565 /**
10566  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10567  * @vsi: PF main vsi
10568  * @seid: seid of main or channel VSIs
10569  *
10570  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10571  * existed before reset
10572  **/
10573 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10574 {
10575         struct i40e_cloud_filter *cfilter;
10576         struct i40e_pf *pf = vsi->back;
10577         struct hlist_node *node;
10578         int ret;
10579
10580         /* Add cloud filters back if they exist */
10581         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10582                                   cloud_node) {
10583                 if (cfilter->seid != seid)
10584                         continue;
10585
10586                 if (cfilter->dst_port)
10587                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10588                                                                 true);
10589                 else
10590                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10591
10592                 if (ret) {
10593                         dev_dbg(&pf->pdev->dev,
10594                                 "Failed to rebuild cloud filter, err %d aq_err %s\n",
10595                                 ret,
10596                                 i40e_aq_str(&pf->hw,
10597                                             pf->hw.aq.asq_last_status));
10598                         return ret;
10599                 }
10600         }
10601         return 0;
10602 }
10603
10604 /**
10605  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10606  * @vsi: PF main vsi
10607  *
10608  * Rebuilds channel VSIs if they existed before reset
10609  **/
10610 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10611 {
10612         struct i40e_channel *ch, *ch_tmp;
10613         int ret;
10614
10615         if (list_empty(&vsi->ch_list))
10616                 return 0;
10617
10618         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10619                 if (!ch->initialized)
10620                         break;
10621                 /* Proceed with creation of channel (VMDq2) VSI */
10622                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10623                 if (ret) {
10624                         dev_info(&vsi->back->pdev->dev,
10625                                  "failed to rebuild channels using uplink_seid %u\n",
10626                                  vsi->uplink_seid);
10627                         return ret;
10628                 }
10629                 /* Reconfigure TX queues using QTX_CTL register */
10630                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10631                 if (ret) {
10632                         dev_info(&vsi->back->pdev->dev,
10633                                  "failed to configure TX rings for channel %u\n",
10634                                  ch->seid);
10635                         return ret;
10636                 }
10637                 /* update 'next_base_queue' */
10638                 vsi->next_base_queue = vsi->next_base_queue +
10639                                                         ch->num_queue_pairs;
10640                 if (ch->max_tx_rate) {
10641                         u64 credits = ch->max_tx_rate;
10642
10643                         if (i40e_set_bw_limit(vsi, ch->seid,
10644                                               ch->max_tx_rate))
10645                                 return -EINVAL;
10646
10647                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
10648                         dev_dbg(&vsi->back->pdev->dev,
10649                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10650                                 ch->max_tx_rate,
10651                                 credits,
10652                                 ch->seid);
10653                 }
10654                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10655                 if (ret) {
10656                         dev_dbg(&vsi->back->pdev->dev,
10657                                 "Failed to rebuild cloud filters for channel VSI %u\n",
10658                                 ch->seid);
10659                         return ret;
10660                 }
10661         }
10662         return 0;
10663 }
10664
10665 /**
10666  * i40e_clean_xps_state - clean xps state for every tx_ring
10667  * @vsi: ptr to the VSI
10668  **/
10669 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10670 {
10671         int i;
10672
10673         if (vsi->tx_rings)
10674                 for (i = 0; i < vsi->num_queue_pairs; i++)
10675                         if (vsi->tx_rings[i])
10676                                 clear_bit(__I40E_TX_XPS_INIT_DONE,
10677                                           vsi->tx_rings[i]->state);
10678 }
10679
10680 /**
10681  * i40e_prep_for_reset - prep for the core to reset
10682  * @pf: board private structure
10683  *
10684  * Close up the VFs and other things in prep for PF Reset.
10685   **/
10686 static void i40e_prep_for_reset(struct i40e_pf *pf)
10687 {
10688         struct i40e_hw *hw = &pf->hw;
10689         int ret = 0;
10690         u32 v;
10691
10692         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10693         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10694                 return;
10695         if (i40e_check_asq_alive(&pf->hw))
10696                 i40e_vc_notify_reset(pf);
10697
10698         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10699
10700         /* quiesce the VSIs and their queues that are not already DOWN */
10701         i40e_pf_quiesce_all_vsi(pf);
10702
10703         for (v = 0; v < pf->num_alloc_vsi; v++) {
10704                 if (pf->vsi[v]) {
10705                         i40e_clean_xps_state(pf->vsi[v]);
10706                         pf->vsi[v]->seid = 0;
10707                 }
10708         }
10709
10710         i40e_shutdown_adminq(&pf->hw);
10711
10712         /* call shutdown HMC */
10713         if (hw->hmc.hmc_obj) {
10714                 ret = i40e_shutdown_lan_hmc(hw);
10715                 if (ret)
10716                         dev_warn(&pf->pdev->dev,
10717                                  "shutdown_lan_hmc failed: %d\n", ret);
10718         }
10719
10720         /* Save the current PTP time so that we can restore the time after the
10721          * reset completes.
10722          */
10723         i40e_ptp_save_hw_time(pf);
10724 }
10725
10726 /**
10727  * i40e_send_version - update firmware with driver version
10728  * @pf: PF struct
10729  */
10730 static void i40e_send_version(struct i40e_pf *pf)
10731 {
10732         struct i40e_driver_version dv;
10733
10734         dv.major_version = 0xff;
10735         dv.minor_version = 0xff;
10736         dv.build_version = 0xff;
10737         dv.subbuild_version = 0;
10738         strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10739         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10740 }
10741
10742 /**
10743  * i40e_get_oem_version - get OEM specific version information
10744  * @hw: pointer to the hardware structure
10745  **/
10746 static void i40e_get_oem_version(struct i40e_hw *hw)
10747 {
10748         u16 block_offset = 0xffff;
10749         u16 block_length = 0;
10750         u16 capabilities = 0;
10751         u16 gen_snap = 0;
10752         u16 release = 0;
10753
10754 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
10755 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
10756 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
10757 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
10758 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
10759 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
10760 #define I40E_NVM_OEM_LENGTH                     3
10761
10762         /* Check if pointer to OEM version block is valid. */
10763         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10764         if (block_offset == 0xffff)
10765                 return;
10766
10767         /* Check if OEM version block has correct length. */
10768         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10769                            &block_length);
10770         if (block_length < I40E_NVM_OEM_LENGTH)
10771                 return;
10772
10773         /* Check if OEM version format is as expected. */
10774         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10775                            &capabilities);
10776         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10777                 return;
10778
10779         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10780                            &gen_snap);
10781         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10782                            &release);
10783         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10784         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10785 }
10786
10787 /**
10788  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10789  * @pf: board private structure
10790  **/
10791 static int i40e_reset(struct i40e_pf *pf)
10792 {
10793         struct i40e_hw *hw = &pf->hw;
10794         int ret;
10795
10796         ret = i40e_pf_reset(hw);
10797         if (ret) {
10798                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10799                 set_bit(__I40E_RESET_FAILED, pf->state);
10800                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10801         } else {
10802                 pf->pfr_count++;
10803         }
10804         return ret;
10805 }
10806
10807 /**
10808  * i40e_rebuild - rebuild using a saved config
10809  * @pf: board private structure
10810  * @reinit: if the Main VSI needs to re-initialized.
10811  * @lock_acquired: indicates whether or not the lock has been acquired
10812  * before this function was called.
10813  **/
10814 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10815 {
10816         const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10817         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10818         struct i40e_hw *hw = &pf->hw;
10819         int ret;
10820         u32 val;
10821         int v;
10822
10823         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10824             is_recovery_mode_reported)
10825                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10826
10827         if (test_bit(__I40E_DOWN, pf->state) &&
10828             !test_bit(__I40E_RECOVERY_MODE, pf->state))
10829                 goto clear_recovery;
10830         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10831
10832         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10833         ret = i40e_init_adminq(&pf->hw);
10834         if (ret) {
10835                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %d aq_err %s\n",
10836                          ret,
10837                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10838                 goto clear_recovery;
10839         }
10840         i40e_get_oem_version(&pf->hw);
10841
10842         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10843                 /* The following delay is necessary for firmware update. */
10844                 mdelay(1000);
10845         }
10846
10847         /* re-verify the eeprom if we just had an EMP reset */
10848         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10849                 i40e_verify_eeprom(pf);
10850
10851         /* if we are going out of or into recovery mode we have to act
10852          * accordingly with regard to resources initialization
10853          * and deinitialization
10854          */
10855         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10856                 if (i40e_get_capabilities(pf,
10857                                           i40e_aqc_opc_list_func_capabilities))
10858                         goto end_unlock;
10859
10860                 if (is_recovery_mode_reported) {
10861                         /* we're staying in recovery mode so we'll reinitialize
10862                          * misc vector here
10863                          */
10864                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10865                                 goto end_unlock;
10866                 } else {
10867                         if (!lock_acquired)
10868                                 rtnl_lock();
10869                         /* we're going out of recovery mode so we'll free
10870                          * the IRQ allocated specifically for recovery mode
10871                          * and restore the interrupt scheme
10872                          */
10873                         free_irq(pf->pdev->irq, pf);
10874                         i40e_clear_interrupt_scheme(pf);
10875                         if (i40e_restore_interrupt_scheme(pf))
10876                                 goto end_unlock;
10877                 }
10878
10879                 /* tell the firmware that we're starting */
10880                 i40e_send_version(pf);
10881
10882                 /* bail out in case recovery mode was detected, as there is
10883                  * no need for further configuration.
10884                  */
10885                 goto end_unlock;
10886         }
10887
10888         i40e_clear_pxe_mode(hw);
10889         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10890         if (ret)
10891                 goto end_core_reset;
10892
10893         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10894                                 hw->func_caps.num_rx_qp, 0, 0);
10895         if (ret) {
10896                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10897                 goto end_core_reset;
10898         }
10899         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10900         if (ret) {
10901                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10902                 goto end_core_reset;
10903         }
10904
10905 #ifdef CONFIG_I40E_DCB
10906         /* Enable FW to write a default DCB config on link-up
10907          * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10908          * is not supported with new link speed
10909          */
10910         if (i40e_is_tc_mqprio_enabled(pf)) {
10911                 i40e_aq_set_dcb_parameters(hw, false, NULL);
10912         } else {
10913                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10914                     (hw->phy.link_info.link_speed &
10915                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10916                         i40e_aq_set_dcb_parameters(hw, false, NULL);
10917                         dev_warn(&pf->pdev->dev,
10918                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10919                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10920                 } else {
10921                         i40e_aq_set_dcb_parameters(hw, true, NULL);
10922                         ret = i40e_init_pf_dcb(pf);
10923                         if (ret) {
10924                                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10925                                          ret);
10926                                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10927                                 /* Continue without DCB enabled */
10928                         }
10929                 }
10930         }
10931
10932 #endif /* CONFIG_I40E_DCB */
10933         if (!lock_acquired)
10934                 rtnl_lock();
10935         ret = i40e_setup_pf_switch(pf, reinit, true);
10936         if (ret)
10937                 goto end_unlock;
10938
10939         /* The driver only wants link up/down and module qualification
10940          * reports from firmware.  Note the negative logic.
10941          */
10942         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10943                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10944                                          I40E_AQ_EVENT_MEDIA_NA |
10945                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10946         if (ret)
10947                 dev_info(&pf->pdev->dev, "set phy mask fail, err %d aq_err %s\n",
10948                          ret,
10949                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10950
10951         /* Rebuild the VSIs and VEBs that existed before reset.
10952          * They are still in our local switch element arrays, so only
10953          * need to rebuild the switch model in the HW.
10954          *
10955          * If there were VEBs but the reconstitution failed, we'll try
10956          * to recover minimal use by getting the basic PF VSI working.
10957          */
10958         if (vsi->uplink_seid != pf->mac_seid) {
10959                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10960                 /* find the one VEB connected to the MAC, and find orphans */
10961                 for (v = 0; v < I40E_MAX_VEB; v++) {
10962                         if (!pf->veb[v])
10963                                 continue;
10964
10965                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10966                             pf->veb[v]->uplink_seid == 0) {
10967                                 ret = i40e_reconstitute_veb(pf->veb[v]);
10968
10969                                 if (!ret)
10970                                         continue;
10971
10972                                 /* If Main VEB failed, we're in deep doodoo,
10973                                  * so give up rebuilding the switch and set up
10974                                  * for minimal rebuild of PF VSI.
10975                                  * If orphan failed, we'll report the error
10976                                  * but try to keep going.
10977                                  */
10978                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10979                                         dev_info(&pf->pdev->dev,
10980                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10981                                                  ret);
10982                                         vsi->uplink_seid = pf->mac_seid;
10983                                         break;
10984                                 } else if (pf->veb[v]->uplink_seid == 0) {
10985                                         dev_info(&pf->pdev->dev,
10986                                                  "rebuild of orphan VEB failed: %d\n",
10987                                                  ret);
10988                                 }
10989                         }
10990                 }
10991         }
10992
10993         if (vsi->uplink_seid == pf->mac_seid) {
10994                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10995                 /* no VEB, so rebuild only the Main VSI */
10996                 ret = i40e_add_vsi(vsi);
10997                 if (ret) {
10998                         dev_info(&pf->pdev->dev,
10999                                  "rebuild of Main VSI failed: %d\n", ret);
11000                         goto end_unlock;
11001                 }
11002         }
11003
11004         if (vsi->mqprio_qopt.max_rate[0]) {
11005                 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11006                                                   vsi->mqprio_qopt.max_rate[0]);
11007                 u64 credits = 0;
11008
11009                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11010                 if (ret)
11011                         goto end_unlock;
11012
11013                 credits = max_tx_rate;
11014                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
11015                 dev_dbg(&vsi->back->pdev->dev,
11016                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11017                         max_tx_rate,
11018                         credits,
11019                         vsi->seid);
11020         }
11021
11022         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11023         if (ret)
11024                 goto end_unlock;
11025
11026         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11027          * for this main VSI if they exist
11028          */
11029         ret = i40e_rebuild_channels(vsi);
11030         if (ret)
11031                 goto end_unlock;
11032
11033         /* Reconfigure hardware for allowing smaller MSS in the case
11034          * of TSO, so that we avoid the MDD being fired and causing
11035          * a reset in the case of small MSS+TSO.
11036          */
11037 #define I40E_REG_MSS          0x000E64DC
11038 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11039 #define I40E_64BYTE_MSS       0x400000
11040         val = rd32(hw, I40E_REG_MSS);
11041         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11042                 val &= ~I40E_REG_MSS_MIN_MASK;
11043                 val |= I40E_64BYTE_MSS;
11044                 wr32(hw, I40E_REG_MSS, val);
11045         }
11046
11047         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
11048                 msleep(75);
11049                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11050                 if (ret)
11051                         dev_info(&pf->pdev->dev, "link restart failed, err %d aq_err %s\n",
11052                                  ret,
11053                                  i40e_aq_str(&pf->hw,
11054                                              pf->hw.aq.asq_last_status));
11055         }
11056         /* reinit the misc interrupt */
11057         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11058                 ret = i40e_setup_misc_vector(pf);
11059                 if (ret)
11060                         goto end_unlock;
11061         }
11062
11063         /* Add a filter to drop all Flow control frames from any VSI from being
11064          * transmitted. By doing so we stop a malicious VF from sending out
11065          * PAUSE or PFC frames and potentially controlling traffic for other
11066          * PF/VF VSIs.
11067          * The FW can still send Flow control frames if enabled.
11068          */
11069         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11070                                                        pf->main_vsi_seid);
11071
11072         /* restart the VSIs that were rebuilt and running before the reset */
11073         i40e_pf_unquiesce_all_vsi(pf);
11074
11075         /* Release the RTNL lock before we start resetting VFs */
11076         if (!lock_acquired)
11077                 rtnl_unlock();
11078
11079         /* Restore promiscuous settings */
11080         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11081         if (ret)
11082                 dev_warn(&pf->pdev->dev,
11083                          "Failed to restore promiscuous setting: %s, err %d aq_err %s\n",
11084                          pf->cur_promisc ? "on" : "off",
11085                          ret,
11086                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11087
11088         i40e_reset_all_vfs(pf, true);
11089
11090         /* tell the firmware that we're starting */
11091         i40e_send_version(pf);
11092
11093         /* We've already released the lock, so don't do it again */
11094         goto end_core_reset;
11095
11096 end_unlock:
11097         if (!lock_acquired)
11098                 rtnl_unlock();
11099 end_core_reset:
11100         clear_bit(__I40E_RESET_FAILED, pf->state);
11101 clear_recovery:
11102         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11103         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11104 }
11105
11106 /**
11107  * i40e_reset_and_rebuild - reset and rebuild using a saved config
11108  * @pf: board private structure
11109  * @reinit: if the Main VSI needs to re-initialized.
11110  * @lock_acquired: indicates whether or not the lock has been acquired
11111  * before this function was called.
11112  **/
11113 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11114                                    bool lock_acquired)
11115 {
11116         int ret;
11117
11118         if (test_bit(__I40E_IN_REMOVE, pf->state))
11119                 return;
11120         /* Now we wait for GRST to settle out.
11121          * We don't have to delete the VEBs or VSIs from the hw switch
11122          * because the reset will make them disappear.
11123          */
11124         ret = i40e_reset(pf);
11125         if (!ret)
11126                 i40e_rebuild(pf, reinit, lock_acquired);
11127 }
11128
11129 /**
11130  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11131  * @pf: board private structure
11132  *
11133  * Close up the VFs and other things in prep for a Core Reset,
11134  * then get ready to rebuild the world.
11135  * @lock_acquired: indicates whether or not the lock has been acquired
11136  * before this function was called.
11137  **/
11138 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11139 {
11140         i40e_prep_for_reset(pf);
11141         i40e_reset_and_rebuild(pf, false, lock_acquired);
11142 }
11143
11144 /**
11145  * i40e_handle_mdd_event
11146  * @pf: pointer to the PF structure
11147  *
11148  * Called from the MDD irq handler to identify possibly malicious vfs
11149  **/
11150 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11151 {
11152         struct i40e_hw *hw = &pf->hw;
11153         bool mdd_detected = false;
11154         struct i40e_vf *vf;
11155         u32 reg;
11156         int i;
11157
11158         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11159                 return;
11160
11161         /* find what triggered the MDD event */
11162         reg = rd32(hw, I40E_GL_MDET_TX);
11163         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11164                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
11165                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
11166                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
11167                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
11168                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
11169                                 I40E_GL_MDET_TX_EVENT_SHIFT;
11170                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
11171                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
11172                                 pf->hw.func_caps.base_queue;
11173                 if (netif_msg_tx_err(pf))
11174                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11175                                  event, queue, pf_num, vf_num);
11176                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11177                 mdd_detected = true;
11178         }
11179         reg = rd32(hw, I40E_GL_MDET_RX);
11180         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11181                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
11182                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
11183                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
11184                                 I40E_GL_MDET_RX_EVENT_SHIFT;
11185                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
11186                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
11187                                 pf->hw.func_caps.base_queue;
11188                 if (netif_msg_rx_err(pf))
11189                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11190                                  event, queue, func);
11191                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11192                 mdd_detected = true;
11193         }
11194
11195         if (mdd_detected) {
11196                 reg = rd32(hw, I40E_PF_MDET_TX);
11197                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11198                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11199                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11200                 }
11201                 reg = rd32(hw, I40E_PF_MDET_RX);
11202                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11203                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11204                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11205                 }
11206         }
11207
11208         /* see if one of the VFs needs its hand slapped */
11209         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11210                 vf = &(pf->vf[i]);
11211                 reg = rd32(hw, I40E_VP_MDET_TX(i));
11212                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11213                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11214                         vf->num_mdd_events++;
11215                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11216                                  i);
11217                         dev_info(&pf->pdev->dev,
11218                                  "Use PF Control I/F to re-enable the VF\n");
11219                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11220                 }
11221
11222                 reg = rd32(hw, I40E_VP_MDET_RX(i));
11223                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11224                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11225                         vf->num_mdd_events++;
11226                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11227                                  i);
11228                         dev_info(&pf->pdev->dev,
11229                                  "Use PF Control I/F to re-enable the VF\n");
11230                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11231                 }
11232         }
11233
11234         /* re-enable mdd interrupt cause */
11235         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
11236         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11237         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11238         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11239         i40e_flush(hw);
11240 }
11241
11242 /**
11243  * i40e_service_task - Run the driver's async subtasks
11244  * @work: pointer to work_struct containing our data
11245  **/
11246 static void i40e_service_task(struct work_struct *work)
11247 {
11248         struct i40e_pf *pf = container_of(work,
11249                                           struct i40e_pf,
11250                                           service_task);
11251         unsigned long start_time = jiffies;
11252
11253         /* don't bother with service tasks if a reset is in progress */
11254         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11255             test_bit(__I40E_SUSPENDED, pf->state))
11256                 return;
11257
11258         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11259                 return;
11260
11261         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11262                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
11263                 i40e_sync_filters_subtask(pf);
11264                 i40e_reset_subtask(pf);
11265                 i40e_handle_mdd_event(pf);
11266                 i40e_vc_process_vflr_event(pf);
11267                 i40e_watchdog_subtask(pf);
11268                 i40e_fdir_reinit_subtask(pf);
11269                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11270                         /* Client subtask will reopen next time through. */
11271                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
11272                                                            true);
11273                 } else {
11274                         i40e_client_subtask(pf);
11275                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11276                                                pf->state))
11277                                 i40e_notify_client_of_l2_param_changes(
11278                                                                 pf->vsi[pf->lan_vsi]);
11279                 }
11280                 i40e_sync_filters_subtask(pf);
11281         } else {
11282                 i40e_reset_subtask(pf);
11283         }
11284
11285         i40e_clean_adminq_subtask(pf);
11286
11287         /* flush memory to make sure state is correct before next watchdog */
11288         smp_mb__before_atomic();
11289         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11290
11291         /* If the tasks have taken longer than one timer cycle or there
11292          * is more work to be done, reschedule the service task now
11293          * rather than wait for the timer to tick again.
11294          */
11295         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11296             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
11297             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
11298             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11299                 i40e_service_event_schedule(pf);
11300 }
11301
11302 /**
11303  * i40e_service_timer - timer callback
11304  * @t: timer list pointer
11305  **/
11306 static void i40e_service_timer(struct timer_list *t)
11307 {
11308         struct i40e_pf *pf = from_timer(pf, t, service_timer);
11309
11310         mod_timer(&pf->service_timer,
11311                   round_jiffies(jiffies + pf->service_timer_period));
11312         i40e_service_event_schedule(pf);
11313 }
11314
11315 /**
11316  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11317  * @vsi: the VSI being configured
11318  **/
11319 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11320 {
11321         struct i40e_pf *pf = vsi->back;
11322
11323         switch (vsi->type) {
11324         case I40E_VSI_MAIN:
11325                 vsi->alloc_queue_pairs = pf->num_lan_qps;
11326                 if (!vsi->num_tx_desc)
11327                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11328                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11329                 if (!vsi->num_rx_desc)
11330                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11331                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11332                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11333                         vsi->num_q_vectors = pf->num_lan_msix;
11334                 else
11335                         vsi->num_q_vectors = 1;
11336
11337                 break;
11338
11339         case I40E_VSI_FDIR:
11340                 vsi->alloc_queue_pairs = 1;
11341                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11342                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11343                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11344                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11345                 vsi->num_q_vectors = pf->num_fdsb_msix;
11346                 break;
11347
11348         case I40E_VSI_VMDQ2:
11349                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11350                 if (!vsi->num_tx_desc)
11351                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11352                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11353                 if (!vsi->num_rx_desc)
11354                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11355                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11356                 vsi->num_q_vectors = pf->num_vmdq_msix;
11357                 break;
11358
11359         case I40E_VSI_SRIOV:
11360                 vsi->alloc_queue_pairs = pf->num_vf_qps;
11361                 if (!vsi->num_tx_desc)
11362                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11363                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11364                 if (!vsi->num_rx_desc)
11365                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11366                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11367                 break;
11368
11369         default:
11370                 WARN_ON(1);
11371                 return -ENODATA;
11372         }
11373
11374         if (is_kdump_kernel()) {
11375                 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11376                 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11377         }
11378
11379         return 0;
11380 }
11381
11382 /**
11383  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11384  * @vsi: VSI pointer
11385  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11386  *
11387  * On error: returns error code (negative)
11388  * On success: returns 0
11389  **/
11390 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11391 {
11392         struct i40e_ring **next_rings;
11393         int size;
11394         int ret = 0;
11395
11396         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11397         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11398                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11399         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11400         if (!vsi->tx_rings)
11401                 return -ENOMEM;
11402         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11403         if (i40e_enabled_xdp_vsi(vsi)) {
11404                 vsi->xdp_rings = next_rings;
11405                 next_rings += vsi->alloc_queue_pairs;
11406         }
11407         vsi->rx_rings = next_rings;
11408
11409         if (alloc_qvectors) {
11410                 /* allocate memory for q_vector pointers */
11411                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11412                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11413                 if (!vsi->q_vectors) {
11414                         ret = -ENOMEM;
11415                         goto err_vectors;
11416                 }
11417         }
11418         return ret;
11419
11420 err_vectors:
11421         kfree(vsi->tx_rings);
11422         return ret;
11423 }
11424
11425 /**
11426  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11427  * @pf: board private structure
11428  * @type: type of VSI
11429  *
11430  * On error: returns error code (negative)
11431  * On success: returns vsi index in PF (positive)
11432  **/
11433 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11434 {
11435         int ret = -ENODEV;
11436         struct i40e_vsi *vsi;
11437         int vsi_idx;
11438         int i;
11439
11440         /* Need to protect the allocation of the VSIs at the PF level */
11441         mutex_lock(&pf->switch_mutex);
11442
11443         /* VSI list may be fragmented if VSI creation/destruction has
11444          * been happening.  We can afford to do a quick scan to look
11445          * for any free VSIs in the list.
11446          *
11447          * find next empty vsi slot, looping back around if necessary
11448          */
11449         i = pf->next_vsi;
11450         while (i < pf->num_alloc_vsi && pf->vsi[i])
11451                 i++;
11452         if (i >= pf->num_alloc_vsi) {
11453                 i = 0;
11454                 while (i < pf->next_vsi && pf->vsi[i])
11455                         i++;
11456         }
11457
11458         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11459                 vsi_idx = i;             /* Found one! */
11460         } else {
11461                 ret = -ENODEV;
11462                 goto unlock_pf;  /* out of VSI slots! */
11463         }
11464         pf->next_vsi = ++i;
11465
11466         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11467         if (!vsi) {
11468                 ret = -ENOMEM;
11469                 goto unlock_pf;
11470         }
11471         vsi->type = type;
11472         vsi->back = pf;
11473         set_bit(__I40E_VSI_DOWN, vsi->state);
11474         vsi->flags = 0;
11475         vsi->idx = vsi_idx;
11476         vsi->int_rate_limit = 0;
11477         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11478                                 pf->rss_table_size : 64;
11479         vsi->netdev_registered = false;
11480         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11481         hash_init(vsi->mac_filter_hash);
11482         vsi->irqs_ready = false;
11483
11484         if (type == I40E_VSI_MAIN) {
11485                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11486                 if (!vsi->af_xdp_zc_qps)
11487                         goto err_rings;
11488         }
11489
11490         ret = i40e_set_num_rings_in_vsi(vsi);
11491         if (ret)
11492                 goto err_rings;
11493
11494         ret = i40e_vsi_alloc_arrays(vsi, true);
11495         if (ret)
11496                 goto err_rings;
11497
11498         /* Setup default MSIX irq handler for VSI */
11499         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11500
11501         /* Initialize VSI lock */
11502         spin_lock_init(&vsi->mac_filter_hash_lock);
11503         pf->vsi[vsi_idx] = vsi;
11504         ret = vsi_idx;
11505         goto unlock_pf;
11506
11507 err_rings:
11508         bitmap_free(vsi->af_xdp_zc_qps);
11509         pf->next_vsi = i - 1;
11510         kfree(vsi);
11511 unlock_pf:
11512         mutex_unlock(&pf->switch_mutex);
11513         return ret;
11514 }
11515
11516 /**
11517  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11518  * @vsi: VSI pointer
11519  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11520  *
11521  * On error: returns error code (negative)
11522  * On success: returns 0
11523  **/
11524 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11525 {
11526         /* free the ring and vector containers */
11527         if (free_qvectors) {
11528                 kfree(vsi->q_vectors);
11529                 vsi->q_vectors = NULL;
11530         }
11531         kfree(vsi->tx_rings);
11532         vsi->tx_rings = NULL;
11533         vsi->rx_rings = NULL;
11534         vsi->xdp_rings = NULL;
11535 }
11536
11537 /**
11538  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11539  * and lookup table
11540  * @vsi: Pointer to VSI structure
11541  */
11542 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11543 {
11544         if (!vsi)
11545                 return;
11546
11547         kfree(vsi->rss_hkey_user);
11548         vsi->rss_hkey_user = NULL;
11549
11550         kfree(vsi->rss_lut_user);
11551         vsi->rss_lut_user = NULL;
11552 }
11553
11554 /**
11555  * i40e_vsi_clear - Deallocate the VSI provided
11556  * @vsi: the VSI being un-configured
11557  **/
11558 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11559 {
11560         struct i40e_pf *pf;
11561
11562         if (!vsi)
11563                 return 0;
11564
11565         if (!vsi->back)
11566                 goto free_vsi;
11567         pf = vsi->back;
11568
11569         mutex_lock(&pf->switch_mutex);
11570         if (!pf->vsi[vsi->idx]) {
11571                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11572                         vsi->idx, vsi->idx, vsi->type);
11573                 goto unlock_vsi;
11574         }
11575
11576         if (pf->vsi[vsi->idx] != vsi) {
11577                 dev_err(&pf->pdev->dev,
11578                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11579                         pf->vsi[vsi->idx]->idx,
11580                         pf->vsi[vsi->idx]->type,
11581                         vsi->idx, vsi->type);
11582                 goto unlock_vsi;
11583         }
11584
11585         /* updates the PF for this cleared vsi */
11586         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11587         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11588
11589         bitmap_free(vsi->af_xdp_zc_qps);
11590         i40e_vsi_free_arrays(vsi, true);
11591         i40e_clear_rss_config_user(vsi);
11592
11593         pf->vsi[vsi->idx] = NULL;
11594         if (vsi->idx < pf->next_vsi)
11595                 pf->next_vsi = vsi->idx;
11596
11597 unlock_vsi:
11598         mutex_unlock(&pf->switch_mutex);
11599 free_vsi:
11600         kfree(vsi);
11601
11602         return 0;
11603 }
11604
11605 /**
11606  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11607  * @vsi: the VSI being cleaned
11608  **/
11609 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11610 {
11611         int i;
11612
11613         if (vsi->tx_rings && vsi->tx_rings[0]) {
11614                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11615                         kfree_rcu(vsi->tx_rings[i], rcu);
11616                         WRITE_ONCE(vsi->tx_rings[i], NULL);
11617                         WRITE_ONCE(vsi->rx_rings[i], NULL);
11618                         if (vsi->xdp_rings)
11619                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11620                 }
11621         }
11622 }
11623
11624 /**
11625  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11626  * @vsi: the VSI being configured
11627  **/
11628 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11629 {
11630         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11631         struct i40e_pf *pf = vsi->back;
11632         struct i40e_ring *ring;
11633
11634         /* Set basic values in the rings to be used later during open() */
11635         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11636                 /* allocate space for both Tx and Rx in one shot */
11637                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11638                 if (!ring)
11639                         goto err_out;
11640
11641                 ring->queue_index = i;
11642                 ring->reg_idx = vsi->base_queue + i;
11643                 ring->ring_active = false;
11644                 ring->vsi = vsi;
11645                 ring->netdev = vsi->netdev;
11646                 ring->dev = &pf->pdev->dev;
11647                 ring->count = vsi->num_tx_desc;
11648                 ring->size = 0;
11649                 ring->dcb_tc = 0;
11650                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11651                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11652                 ring->itr_setting = pf->tx_itr_default;
11653                 WRITE_ONCE(vsi->tx_rings[i], ring++);
11654
11655                 if (!i40e_enabled_xdp_vsi(vsi))
11656                         goto setup_rx;
11657
11658                 ring->queue_index = vsi->alloc_queue_pairs + i;
11659                 ring->reg_idx = vsi->base_queue + ring->queue_index;
11660                 ring->ring_active = false;
11661                 ring->vsi = vsi;
11662                 ring->netdev = NULL;
11663                 ring->dev = &pf->pdev->dev;
11664                 ring->count = vsi->num_tx_desc;
11665                 ring->size = 0;
11666                 ring->dcb_tc = 0;
11667                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11668                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11669                 set_ring_xdp(ring);
11670                 ring->itr_setting = pf->tx_itr_default;
11671                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11672
11673 setup_rx:
11674                 ring->queue_index = i;
11675                 ring->reg_idx = vsi->base_queue + i;
11676                 ring->ring_active = false;
11677                 ring->vsi = vsi;
11678                 ring->netdev = vsi->netdev;
11679                 ring->dev = &pf->pdev->dev;
11680                 ring->count = vsi->num_rx_desc;
11681                 ring->size = 0;
11682                 ring->dcb_tc = 0;
11683                 ring->itr_setting = pf->rx_itr_default;
11684                 WRITE_ONCE(vsi->rx_rings[i], ring);
11685         }
11686
11687         return 0;
11688
11689 err_out:
11690         i40e_vsi_clear_rings(vsi);
11691         return -ENOMEM;
11692 }
11693
11694 /**
11695  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11696  * @pf: board private structure
11697  * @vectors: the number of MSI-X vectors to request
11698  *
11699  * Returns the number of vectors reserved, or error
11700  **/
11701 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11702 {
11703         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11704                                         I40E_MIN_MSIX, vectors);
11705         if (vectors < 0) {
11706                 dev_info(&pf->pdev->dev,
11707                          "MSI-X vector reservation failed: %d\n", vectors);
11708                 vectors = 0;
11709         }
11710
11711         return vectors;
11712 }
11713
11714 /**
11715  * i40e_init_msix - Setup the MSIX capability
11716  * @pf: board private structure
11717  *
11718  * Work with the OS to set up the MSIX vectors needed.
11719  *
11720  * Returns the number of vectors reserved or negative on failure
11721  **/
11722 static int i40e_init_msix(struct i40e_pf *pf)
11723 {
11724         struct i40e_hw *hw = &pf->hw;
11725         int cpus, extra_vectors;
11726         int vectors_left;
11727         int v_budget, i;
11728         int v_actual;
11729         int iwarp_requested = 0;
11730
11731         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11732                 return -ENODEV;
11733
11734         /* The number of vectors we'll request will be comprised of:
11735          *   - Add 1 for "other" cause for Admin Queue events, etc.
11736          *   - The number of LAN queue pairs
11737          *      - Queues being used for RSS.
11738          *              We don't need as many as max_rss_size vectors.
11739          *              use rss_size instead in the calculation since that
11740          *              is governed by number of cpus in the system.
11741          *      - assumes symmetric Tx/Rx pairing
11742          *   - The number of VMDq pairs
11743          *   - The CPU count within the NUMA node if iWARP is enabled
11744          * Once we count this up, try the request.
11745          *
11746          * If we can't get what we want, we'll simplify to nearly nothing
11747          * and try again.  If that still fails, we punt.
11748          */
11749         vectors_left = hw->func_caps.num_msix_vectors;
11750         v_budget = 0;
11751
11752         /* reserve one vector for miscellaneous handler */
11753         if (vectors_left) {
11754                 v_budget++;
11755                 vectors_left--;
11756         }
11757
11758         /* reserve some vectors for the main PF traffic queues. Initially we
11759          * only reserve at most 50% of the available vectors, in the case that
11760          * the number of online CPUs is large. This ensures that we can enable
11761          * extra features as well. Once we've enabled the other features, we
11762          * will use any remaining vectors to reach as close as we can to the
11763          * number of online CPUs.
11764          */
11765         cpus = num_online_cpus();
11766         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11767         vectors_left -= pf->num_lan_msix;
11768
11769         /* reserve one vector for sideband flow director */
11770         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11771                 if (vectors_left) {
11772                         pf->num_fdsb_msix = 1;
11773                         v_budget++;
11774                         vectors_left--;
11775                 } else {
11776                         pf->num_fdsb_msix = 0;
11777                 }
11778         }
11779
11780         /* can we reserve enough for iWARP? */
11781         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11782                 iwarp_requested = pf->num_iwarp_msix;
11783
11784                 if (!vectors_left)
11785                         pf->num_iwarp_msix = 0;
11786                 else if (vectors_left < pf->num_iwarp_msix)
11787                         pf->num_iwarp_msix = 1;
11788                 v_budget += pf->num_iwarp_msix;
11789                 vectors_left -= pf->num_iwarp_msix;
11790         }
11791
11792         /* any vectors left over go for VMDq support */
11793         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11794                 if (!vectors_left) {
11795                         pf->num_vmdq_msix = 0;
11796                         pf->num_vmdq_qps = 0;
11797                 } else {
11798                         int vmdq_vecs_wanted =
11799                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11800                         int vmdq_vecs =
11801                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11802
11803                         /* if we're short on vectors for what's desired, we limit
11804                          * the queues per vmdq.  If this is still more than are
11805                          * available, the user will need to change the number of
11806                          * queues/vectors used by the PF later with the ethtool
11807                          * channels command
11808                          */
11809                         if (vectors_left < vmdq_vecs_wanted) {
11810                                 pf->num_vmdq_qps = 1;
11811                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11812                                 vmdq_vecs = min_t(int,
11813                                                   vectors_left,
11814                                                   vmdq_vecs_wanted);
11815                         }
11816                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11817
11818                         v_budget += vmdq_vecs;
11819                         vectors_left -= vmdq_vecs;
11820                 }
11821         }
11822
11823         /* On systems with a large number of SMP cores, we previously limited
11824          * the number of vectors for num_lan_msix to be at most 50% of the
11825          * available vectors, to allow for other features. Now, we add back
11826          * the remaining vectors. However, we ensure that the total
11827          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11828          * calculate the number of vectors we can add without going over the
11829          * cap of CPUs. For systems with a small number of CPUs this will be
11830          * zero.
11831          */
11832         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11833         pf->num_lan_msix += extra_vectors;
11834         vectors_left -= extra_vectors;
11835
11836         WARN(vectors_left < 0,
11837              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11838
11839         v_budget += pf->num_lan_msix;
11840         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11841                                    GFP_KERNEL);
11842         if (!pf->msix_entries)
11843                 return -ENOMEM;
11844
11845         for (i = 0; i < v_budget; i++)
11846                 pf->msix_entries[i].entry = i;
11847         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11848
11849         if (v_actual < I40E_MIN_MSIX) {
11850                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11851                 kfree(pf->msix_entries);
11852                 pf->msix_entries = NULL;
11853                 pci_disable_msix(pf->pdev);
11854                 return -ENODEV;
11855
11856         } else if (v_actual == I40E_MIN_MSIX) {
11857                 /* Adjust for minimal MSIX use */
11858                 pf->num_vmdq_vsis = 0;
11859                 pf->num_vmdq_qps = 0;
11860                 pf->num_lan_qps = 1;
11861                 pf->num_lan_msix = 1;
11862
11863         } else if (v_actual != v_budget) {
11864                 /* If we have limited resources, we will start with no vectors
11865                  * for the special features and then allocate vectors to some
11866                  * of these features based on the policy and at the end disable
11867                  * the features that did not get any vectors.
11868                  */
11869                 int vec;
11870
11871                 dev_info(&pf->pdev->dev,
11872                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11873                          v_actual, v_budget);
11874                 /* reserve the misc vector */
11875                 vec = v_actual - 1;
11876
11877                 /* Scale vector usage down */
11878                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11879                 pf->num_vmdq_vsis = 1;
11880                 pf->num_vmdq_qps = 1;
11881
11882                 /* partition out the remaining vectors */
11883                 switch (vec) {
11884                 case 2:
11885                         pf->num_lan_msix = 1;
11886                         break;
11887                 case 3:
11888                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11889                                 pf->num_lan_msix = 1;
11890                                 pf->num_iwarp_msix = 1;
11891                         } else {
11892                                 pf->num_lan_msix = 2;
11893                         }
11894                         break;
11895                 default:
11896                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11897                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11898                                                  iwarp_requested);
11899                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11900                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11901                         } else {
11902                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11903                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11904                         }
11905                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11906                                 pf->num_fdsb_msix = 1;
11907                                 vec--;
11908                         }
11909                         pf->num_lan_msix = min_t(int,
11910                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11911                                                               pf->num_lan_msix);
11912                         pf->num_lan_qps = pf->num_lan_msix;
11913                         break;
11914                 }
11915         }
11916
11917         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11918             (pf->num_fdsb_msix == 0)) {
11919                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11920                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11921                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11922         }
11923         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11924             (pf->num_vmdq_msix == 0)) {
11925                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11926                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11927         }
11928
11929         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11930             (pf->num_iwarp_msix == 0)) {
11931                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11932                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11933         }
11934         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11935                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11936                    pf->num_lan_msix,
11937                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11938                    pf->num_fdsb_msix,
11939                    pf->num_iwarp_msix);
11940
11941         return v_actual;
11942 }
11943
11944 /**
11945  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11946  * @vsi: the VSI being configured
11947  * @v_idx: index of the vector in the vsi struct
11948  *
11949  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11950  **/
11951 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11952 {
11953         struct i40e_q_vector *q_vector;
11954
11955         /* allocate q_vector */
11956         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11957         if (!q_vector)
11958                 return -ENOMEM;
11959
11960         q_vector->vsi = vsi;
11961         q_vector->v_idx = v_idx;
11962         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11963
11964         if (vsi->netdev)
11965                 netif_napi_add(vsi->netdev, &q_vector->napi,
11966                                i40e_napi_poll, NAPI_POLL_WEIGHT);
11967
11968         /* tie q_vector and vsi together */
11969         vsi->q_vectors[v_idx] = q_vector;
11970
11971         return 0;
11972 }
11973
11974 /**
11975  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11976  * @vsi: the VSI being configured
11977  *
11978  * We allocate one q_vector per queue interrupt.  If allocation fails we
11979  * return -ENOMEM.
11980  **/
11981 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11982 {
11983         struct i40e_pf *pf = vsi->back;
11984         int err, v_idx, num_q_vectors;
11985
11986         /* if not MSIX, give the one vector only to the LAN VSI */
11987         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11988                 num_q_vectors = vsi->num_q_vectors;
11989         else if (vsi == pf->vsi[pf->lan_vsi])
11990                 num_q_vectors = 1;
11991         else
11992                 return -EINVAL;
11993
11994         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11995                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
11996                 if (err)
11997                         goto err_out;
11998         }
11999
12000         return 0;
12001
12002 err_out:
12003         while (v_idx--)
12004                 i40e_free_q_vector(vsi, v_idx);
12005
12006         return err;
12007 }
12008
12009 /**
12010  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12011  * @pf: board private structure to initialize
12012  **/
12013 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12014 {
12015         int vectors = 0;
12016         ssize_t size;
12017
12018         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12019                 vectors = i40e_init_msix(pf);
12020                 if (vectors < 0) {
12021                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
12022                                        I40E_FLAG_IWARP_ENABLED  |
12023                                        I40E_FLAG_RSS_ENABLED    |
12024                                        I40E_FLAG_DCB_CAPABLE    |
12025                                        I40E_FLAG_DCB_ENABLED    |
12026                                        I40E_FLAG_SRIOV_ENABLED  |
12027                                        I40E_FLAG_FD_SB_ENABLED  |
12028                                        I40E_FLAG_FD_ATR_ENABLED |
12029                                        I40E_FLAG_VMDQ_ENABLED);
12030                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12031
12032                         /* rework the queue expectations without MSIX */
12033                         i40e_determine_queue_usage(pf);
12034                 }
12035         }
12036
12037         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
12038             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
12039                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12040                 vectors = pci_enable_msi(pf->pdev);
12041                 if (vectors < 0) {
12042                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12043                                  vectors);
12044                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
12045                 }
12046                 vectors = 1;  /* one MSI or Legacy vector */
12047         }
12048
12049         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
12050                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12051
12052         /* set up vector assignment tracking */
12053         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12054         pf->irq_pile = kzalloc(size, GFP_KERNEL);
12055         if (!pf->irq_pile)
12056                 return -ENOMEM;
12057
12058         pf->irq_pile->num_entries = vectors;
12059
12060         /* track first vector for misc interrupts, ignore return */
12061         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12062
12063         return 0;
12064 }
12065
12066 /**
12067  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12068  * @pf: private board data structure
12069  *
12070  * Restore the interrupt scheme that was cleared when we suspended the
12071  * device. This should be called during resume to re-allocate the q_vectors
12072  * and reacquire IRQs.
12073  */
12074 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12075 {
12076         int err, i;
12077
12078         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
12079          * scheme. We need to re-enabled them here in order to attempt to
12080          * re-acquire the MSI or MSI-X vectors
12081          */
12082         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
12083
12084         err = i40e_init_interrupt_scheme(pf);
12085         if (err)
12086                 return err;
12087
12088         /* Now that we've re-acquired IRQs, we need to remap the vectors and
12089          * rings together again.
12090          */
12091         for (i = 0; i < pf->num_alloc_vsi; i++) {
12092                 if (pf->vsi[i]) {
12093                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
12094                         if (err)
12095                                 goto err_unwind;
12096                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
12097                 }
12098         }
12099
12100         err = i40e_setup_misc_vector(pf);
12101         if (err)
12102                 goto err_unwind;
12103
12104         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
12105                 i40e_client_update_msix_info(pf);
12106
12107         return 0;
12108
12109 err_unwind:
12110         while (i--) {
12111                 if (pf->vsi[i])
12112                         i40e_vsi_free_q_vectors(pf->vsi[i]);
12113         }
12114
12115         return err;
12116 }
12117
12118 /**
12119  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12120  * non queue events in recovery mode
12121  * @pf: board private structure
12122  *
12123  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12124  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12125  * This is handled differently than in recovery mode since no Tx/Rx resources
12126  * are being allocated.
12127  **/
12128 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12129 {
12130         int err;
12131
12132         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12133                 err = i40e_setup_misc_vector(pf);
12134
12135                 if (err) {
12136                         dev_info(&pf->pdev->dev,
12137                                  "MSI-X misc vector request failed, error %d\n",
12138                                  err);
12139                         return err;
12140                 }
12141         } else {
12142                 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
12143
12144                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
12145                                   pf->int_name, pf);
12146
12147                 if (err) {
12148                         dev_info(&pf->pdev->dev,
12149                                  "MSI/legacy misc vector request failed, error %d\n",
12150                                  err);
12151                         return err;
12152                 }
12153                 i40e_enable_misc_int_causes(pf);
12154                 i40e_irq_dynamic_enable_icr0(pf);
12155         }
12156
12157         return 0;
12158 }
12159
12160 /**
12161  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12162  * @pf: board private structure
12163  *
12164  * This sets up the handler for MSIX 0, which is used to manage the
12165  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
12166  * when in MSI or Legacy interrupt mode.
12167  **/
12168 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12169 {
12170         struct i40e_hw *hw = &pf->hw;
12171         int err = 0;
12172
12173         /* Only request the IRQ once, the first time through. */
12174         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12175                 err = request_irq(pf->msix_entries[0].vector,
12176                                   i40e_intr, 0, pf->int_name, pf);
12177                 if (err) {
12178                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12179                         dev_info(&pf->pdev->dev,
12180                                  "request_irq for %s failed: %d\n",
12181                                  pf->int_name, err);
12182                         return -EFAULT;
12183                 }
12184         }
12185
12186         i40e_enable_misc_int_causes(pf);
12187
12188         /* associate no queues to the misc vector */
12189         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12190         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12191
12192         i40e_flush(hw);
12193
12194         i40e_irq_dynamic_enable_icr0(pf);
12195
12196         return err;
12197 }
12198
12199 /**
12200  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12201  * @vsi: Pointer to vsi structure
12202  * @seed: Buffter to store the hash keys
12203  * @lut: Buffer to store the lookup table entries
12204  * @lut_size: Size of buffer to store the lookup table entries
12205  *
12206  * Return 0 on success, negative on failure
12207  */
12208 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12209                            u8 *lut, u16 lut_size)
12210 {
12211         struct i40e_pf *pf = vsi->back;
12212         struct i40e_hw *hw = &pf->hw;
12213         int ret = 0;
12214
12215         if (seed) {
12216                 ret = i40e_aq_get_rss_key(hw, vsi->id,
12217                         (struct i40e_aqc_get_set_rss_key_data *)seed);
12218                 if (ret) {
12219                         dev_info(&pf->pdev->dev,
12220                                  "Cannot get RSS key, err %d aq_err %s\n",
12221                                  ret,
12222                                  i40e_aq_str(&pf->hw,
12223                                              pf->hw.aq.asq_last_status));
12224                         return ret;
12225                 }
12226         }
12227
12228         if (lut) {
12229                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
12230
12231                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12232                 if (ret) {
12233                         dev_info(&pf->pdev->dev,
12234                                  "Cannot get RSS lut, err %d aq_err %s\n",
12235                                  ret,
12236                                  i40e_aq_str(&pf->hw,
12237                                              pf->hw.aq.asq_last_status));
12238                         return ret;
12239                 }
12240         }
12241
12242         return ret;
12243 }
12244
12245 /**
12246  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12247  * @vsi: Pointer to vsi structure
12248  * @seed: RSS hash seed
12249  * @lut: Lookup table
12250  * @lut_size: Lookup table size
12251  *
12252  * Returns 0 on success, negative on failure
12253  **/
12254 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12255                                const u8 *lut, u16 lut_size)
12256 {
12257         struct i40e_pf *pf = vsi->back;
12258         struct i40e_hw *hw = &pf->hw;
12259         u16 vf_id = vsi->vf_id;
12260         u8 i;
12261
12262         /* Fill out hash function seed */
12263         if (seed) {
12264                 u32 *seed_dw = (u32 *)seed;
12265
12266                 if (vsi->type == I40E_VSI_MAIN) {
12267                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12268                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12269                 } else if (vsi->type == I40E_VSI_SRIOV) {
12270                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12271                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12272                 } else {
12273                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12274                 }
12275         }
12276
12277         if (lut) {
12278                 u32 *lut_dw = (u32 *)lut;
12279
12280                 if (vsi->type == I40E_VSI_MAIN) {
12281                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
12282                                 return -EINVAL;
12283                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12284                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12285                 } else if (vsi->type == I40E_VSI_SRIOV) {
12286                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12287                                 return -EINVAL;
12288                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12289                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12290                 } else {
12291                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12292                 }
12293         }
12294         i40e_flush(hw);
12295
12296         return 0;
12297 }
12298
12299 /**
12300  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12301  * @vsi: Pointer to VSI structure
12302  * @seed: Buffer to store the keys
12303  * @lut: Buffer to store the lookup table entries
12304  * @lut_size: Size of buffer to store the lookup table entries
12305  *
12306  * Returns 0 on success, negative on failure
12307  */
12308 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12309                             u8 *lut, u16 lut_size)
12310 {
12311         struct i40e_pf *pf = vsi->back;
12312         struct i40e_hw *hw = &pf->hw;
12313         u16 i;
12314
12315         if (seed) {
12316                 u32 *seed_dw = (u32 *)seed;
12317
12318                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12319                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12320         }
12321         if (lut) {
12322                 u32 *lut_dw = (u32 *)lut;
12323
12324                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12325                         return -EINVAL;
12326                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12327                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12328         }
12329
12330         return 0;
12331 }
12332
12333 /**
12334  * i40e_config_rss - Configure RSS keys and lut
12335  * @vsi: Pointer to VSI structure
12336  * @seed: RSS hash seed
12337  * @lut: Lookup table
12338  * @lut_size: Lookup table size
12339  *
12340  * Returns 0 on success, negative on failure
12341  */
12342 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12343 {
12344         struct i40e_pf *pf = vsi->back;
12345
12346         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12347                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12348         else
12349                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12350 }
12351
12352 /**
12353  * i40e_get_rss - Get RSS keys and lut
12354  * @vsi: Pointer to VSI structure
12355  * @seed: Buffer to store the keys
12356  * @lut: Buffer to store the lookup table entries
12357  * @lut_size: Size of buffer to store the lookup table entries
12358  *
12359  * Returns 0 on success, negative on failure
12360  */
12361 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12362 {
12363         struct i40e_pf *pf = vsi->back;
12364
12365         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12366                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12367         else
12368                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12369 }
12370
12371 /**
12372  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12373  * @pf: Pointer to board private structure
12374  * @lut: Lookup table
12375  * @rss_table_size: Lookup table size
12376  * @rss_size: Range of queue number for hashing
12377  */
12378 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12379                        u16 rss_table_size, u16 rss_size)
12380 {
12381         u16 i;
12382
12383         for (i = 0; i < rss_table_size; i++)
12384                 lut[i] = i % rss_size;
12385 }
12386
12387 /**
12388  * i40e_pf_config_rss - Prepare for RSS if used
12389  * @pf: board private structure
12390  **/
12391 static int i40e_pf_config_rss(struct i40e_pf *pf)
12392 {
12393         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12394         u8 seed[I40E_HKEY_ARRAY_SIZE];
12395         u8 *lut;
12396         struct i40e_hw *hw = &pf->hw;
12397         u32 reg_val;
12398         u64 hena;
12399         int ret;
12400
12401         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12402         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12403                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12404         hena |= i40e_pf_get_default_rss_hena(pf);
12405
12406         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12407         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12408
12409         /* Determine the RSS table size based on the hardware capabilities */
12410         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12411         reg_val = (pf->rss_table_size == 512) ?
12412                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12413                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12414         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12415
12416         /* Determine the RSS size of the VSI */
12417         if (!vsi->rss_size) {
12418                 u16 qcount;
12419                 /* If the firmware does something weird during VSI init, we
12420                  * could end up with zero TCs. Check for that to avoid
12421                  * divide-by-zero. It probably won't pass traffic, but it also
12422                  * won't panic.
12423                  */
12424                 qcount = vsi->num_queue_pairs /
12425                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12426                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12427         }
12428         if (!vsi->rss_size)
12429                 return -EINVAL;
12430
12431         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12432         if (!lut)
12433                 return -ENOMEM;
12434
12435         /* Use user configured lut if there is one, otherwise use default */
12436         if (vsi->rss_lut_user)
12437                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12438         else
12439                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12440
12441         /* Use user configured hash key if there is one, otherwise
12442          * use default.
12443          */
12444         if (vsi->rss_hkey_user)
12445                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12446         else
12447                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12448         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12449         kfree(lut);
12450
12451         return ret;
12452 }
12453
12454 /**
12455  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12456  * @pf: board private structure
12457  * @queue_count: the requested queue count for rss.
12458  *
12459  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12460  * count which may be different from the requested queue count.
12461  * Note: expects to be called while under rtnl_lock()
12462  **/
12463 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12464 {
12465         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12466         int new_rss_size;
12467
12468         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
12469                 return 0;
12470
12471         queue_count = min_t(int, queue_count, num_online_cpus());
12472         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12473
12474         if (queue_count != vsi->num_queue_pairs) {
12475                 u16 qcount;
12476
12477                 vsi->req_queue_pairs = queue_count;
12478                 i40e_prep_for_reset(pf);
12479                 if (test_bit(__I40E_IN_REMOVE, pf->state))
12480                         return pf->alloc_rss_size;
12481
12482                 pf->alloc_rss_size = new_rss_size;
12483
12484                 i40e_reset_and_rebuild(pf, true, true);
12485
12486                 /* Discard the user configured hash keys and lut, if less
12487                  * queues are enabled.
12488                  */
12489                 if (queue_count < vsi->rss_size) {
12490                         i40e_clear_rss_config_user(vsi);
12491                         dev_dbg(&pf->pdev->dev,
12492                                 "discard user configured hash keys and lut\n");
12493                 }
12494
12495                 /* Reset vsi->rss_size, as number of enabled queues changed */
12496                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12497                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12498
12499                 i40e_pf_config_rss(pf);
12500         }
12501         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12502                  vsi->req_queue_pairs, pf->rss_size_max);
12503         return pf->alloc_rss_size;
12504 }
12505
12506 /**
12507  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12508  * @pf: board private structure
12509  **/
12510 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12511 {
12512         bool min_valid, max_valid;
12513         u32 max_bw, min_bw;
12514         int status;
12515
12516         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12517                                            &min_valid, &max_valid);
12518
12519         if (!status) {
12520                 if (min_valid)
12521                         pf->min_bw = min_bw;
12522                 if (max_valid)
12523                         pf->max_bw = max_bw;
12524         }
12525
12526         return status;
12527 }
12528
12529 /**
12530  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12531  * @pf: board private structure
12532  **/
12533 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12534 {
12535         struct i40e_aqc_configure_partition_bw_data bw_data;
12536         int status;
12537
12538         memset(&bw_data, 0, sizeof(bw_data));
12539
12540         /* Set the valid bit for this PF */
12541         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12542         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12543         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12544
12545         /* Set the new bandwidths */
12546         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12547
12548         return status;
12549 }
12550
12551 /**
12552  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12553  * @pf: board private structure
12554  **/
12555 int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12556 {
12557         /* Commit temporary BW setting to permanent NVM image */
12558         enum i40e_admin_queue_err last_aq_status;
12559         u16 nvm_word;
12560         int ret;
12561
12562         if (pf->hw.partition_id != 1) {
12563                 dev_info(&pf->pdev->dev,
12564                          "Commit BW only works on partition 1! This is partition %d",
12565                          pf->hw.partition_id);
12566                 ret = I40E_NOT_SUPPORTED;
12567                 goto bw_commit_out;
12568         }
12569
12570         /* Acquire NVM for read access */
12571         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12572         last_aq_status = pf->hw.aq.asq_last_status;
12573         if (ret) {
12574                 dev_info(&pf->pdev->dev,
12575                          "Cannot acquire NVM for read access, err %d aq_err %s\n",
12576                          ret,
12577                          i40e_aq_str(&pf->hw, last_aq_status));
12578                 goto bw_commit_out;
12579         }
12580
12581         /* Read word 0x10 of NVM - SW compatibility word 1 */
12582         ret = i40e_aq_read_nvm(&pf->hw,
12583                                I40E_SR_NVM_CONTROL_WORD,
12584                                0x10, sizeof(nvm_word), &nvm_word,
12585                                false, NULL);
12586         /* Save off last admin queue command status before releasing
12587          * the NVM
12588          */
12589         last_aq_status = pf->hw.aq.asq_last_status;
12590         i40e_release_nvm(&pf->hw);
12591         if (ret) {
12592                 dev_info(&pf->pdev->dev, "NVM read error, err %d aq_err %s\n",
12593                          ret,
12594                          i40e_aq_str(&pf->hw, last_aq_status));
12595                 goto bw_commit_out;
12596         }
12597
12598         /* Wait a bit for NVM release to complete */
12599         msleep(50);
12600
12601         /* Acquire NVM for write access */
12602         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12603         last_aq_status = pf->hw.aq.asq_last_status;
12604         if (ret) {
12605                 dev_info(&pf->pdev->dev,
12606                          "Cannot acquire NVM for write access, err %d aq_err %s\n",
12607                          ret,
12608                          i40e_aq_str(&pf->hw, last_aq_status));
12609                 goto bw_commit_out;
12610         }
12611         /* Write it back out unchanged to initiate update NVM,
12612          * which will force a write of the shadow (alt) RAM to
12613          * the NVM - thus storing the bandwidth values permanently.
12614          */
12615         ret = i40e_aq_update_nvm(&pf->hw,
12616                                  I40E_SR_NVM_CONTROL_WORD,
12617                                  0x10, sizeof(nvm_word),
12618                                  &nvm_word, true, 0, NULL);
12619         /* Save off last admin queue command status before releasing
12620          * the NVM
12621          */
12622         last_aq_status = pf->hw.aq.asq_last_status;
12623         i40e_release_nvm(&pf->hw);
12624         if (ret)
12625                 dev_info(&pf->pdev->dev,
12626                          "BW settings NOT SAVED, err %d aq_err %s\n",
12627                          ret,
12628                          i40e_aq_str(&pf->hw, last_aq_status));
12629 bw_commit_out:
12630
12631         return ret;
12632 }
12633
12634 /**
12635  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12636  * if total port shutdown feature is enabled for this PF
12637  * @pf: board private structure
12638  **/
12639 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12640 {
12641 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED        BIT(4)
12642 #define I40E_FEATURES_ENABLE_PTR                0x2A
12643 #define I40E_CURRENT_SETTING_PTR                0x2B
12644 #define I40E_LINK_BEHAVIOR_WORD_OFFSET          0x2D
12645 #define I40E_LINK_BEHAVIOR_WORD_LENGTH          0x1
12646 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED    BIT(0)
12647 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH      4
12648         int read_status = I40E_SUCCESS;
12649         u16 sr_emp_sr_settings_ptr = 0;
12650         u16 features_enable = 0;
12651         u16 link_behavior = 0;
12652         bool ret = false;
12653
12654         read_status = i40e_read_nvm_word(&pf->hw,
12655                                          I40E_SR_EMP_SR_SETTINGS_PTR,
12656                                          &sr_emp_sr_settings_ptr);
12657         if (read_status)
12658                 goto err_nvm;
12659         read_status = i40e_read_nvm_word(&pf->hw,
12660                                          sr_emp_sr_settings_ptr +
12661                                          I40E_FEATURES_ENABLE_PTR,
12662                                          &features_enable);
12663         if (read_status)
12664                 goto err_nvm;
12665         if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12666                 read_status = i40e_read_nvm_module_data(&pf->hw,
12667                                                         I40E_SR_EMP_SR_SETTINGS_PTR,
12668                                                         I40E_CURRENT_SETTING_PTR,
12669                                                         I40E_LINK_BEHAVIOR_WORD_OFFSET,
12670                                                         I40E_LINK_BEHAVIOR_WORD_LENGTH,
12671                                                         &link_behavior);
12672                 if (read_status)
12673                         goto err_nvm;
12674                 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12675                 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12676         }
12677         return ret;
12678
12679 err_nvm:
12680         dev_warn(&pf->pdev->dev,
12681                  "total-port-shutdown feature is off due to read nvm error: %d\n",
12682                  read_status);
12683         return ret;
12684 }
12685
12686 /**
12687  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12688  * @pf: board private structure to initialize
12689  *
12690  * i40e_sw_init initializes the Adapter private data structure.
12691  * Fields are initialized based on PCI device information and
12692  * OS network device settings (MTU size).
12693  **/
12694 static int i40e_sw_init(struct i40e_pf *pf)
12695 {
12696         int err = 0;
12697         int size;
12698         u16 pow;
12699
12700         /* Set default capability flags */
12701         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12702                     I40E_FLAG_MSI_ENABLED     |
12703                     I40E_FLAG_MSIX_ENABLED;
12704
12705         /* Set default ITR */
12706         pf->rx_itr_default = I40E_ITR_RX_DEF;
12707         pf->tx_itr_default = I40E_ITR_TX_DEF;
12708
12709         /* Depending on PF configurations, it is possible that the RSS
12710          * maximum might end up larger than the available queues
12711          */
12712         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12713         pf->alloc_rss_size = 1;
12714         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12715         pf->rss_size_max = min_t(int, pf->rss_size_max,
12716                                  pf->hw.func_caps.num_tx_qp);
12717
12718         /* find the next higher power-of-2 of num cpus */
12719         pow = roundup_pow_of_two(num_online_cpus());
12720         pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12721
12722         if (pf->hw.func_caps.rss) {
12723                 pf->flags |= I40E_FLAG_RSS_ENABLED;
12724                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12725                                            num_online_cpus());
12726         }
12727
12728         /* MFP mode enabled */
12729         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12730                 pf->flags |= I40E_FLAG_MFP_ENABLED;
12731                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12732                 if (i40e_get_partition_bw_setting(pf)) {
12733                         dev_warn(&pf->pdev->dev,
12734                                  "Could not get partition bw settings\n");
12735                 } else {
12736                         dev_info(&pf->pdev->dev,
12737                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
12738                                  pf->min_bw, pf->max_bw);
12739
12740                         /* nudge the Tx scheduler */
12741                         i40e_set_partition_bw_setting(pf);
12742                 }
12743         }
12744
12745         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12746             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12747                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12748                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12749                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12750                     pf->hw.num_partitions > 1)
12751                         dev_info(&pf->pdev->dev,
12752                                  "Flow Director Sideband mode Disabled in MFP mode\n");
12753                 else
12754                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12755                 pf->fdir_pf_filter_count =
12756                                  pf->hw.func_caps.fd_filters_guaranteed;
12757                 pf->hw.fdir_shared_filter_count =
12758                                  pf->hw.func_caps.fd_filters_best_effort;
12759         }
12760
12761         if (pf->hw.mac.type == I40E_MAC_X722) {
12762                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12763                                     I40E_HW_128_QP_RSS_CAPABLE |
12764                                     I40E_HW_ATR_EVICT_CAPABLE |
12765                                     I40E_HW_WB_ON_ITR_CAPABLE |
12766                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12767                                     I40E_HW_NO_PCI_LINK_CHECK |
12768                                     I40E_HW_USE_SET_LLDP_MIB |
12769                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12770                                     I40E_HW_PTP_L4_CAPABLE |
12771                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12772                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12773
12774 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12775                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12776                     I40E_FDEVICT_PCTYPE_DEFAULT) {
12777                         dev_warn(&pf->pdev->dev,
12778                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12779                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12780                 }
12781         } else if ((pf->hw.aq.api_maj_ver > 1) ||
12782                    ((pf->hw.aq.api_maj_ver == 1) &&
12783                     (pf->hw.aq.api_min_ver > 4))) {
12784                 /* Supported in FW API version higher than 1.4 */
12785                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12786         }
12787
12788         /* Enable HW ATR eviction if possible */
12789         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12790                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12791
12792         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12793             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12794             (pf->hw.aq.fw_maj_ver < 4))) {
12795                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12796                 /* No DCB support  for FW < v4.33 */
12797                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12798         }
12799
12800         /* Disable FW LLDP if FW < v4.3 */
12801         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12802             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12803             (pf->hw.aq.fw_maj_ver < 4)))
12804                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12805
12806         /* Use the FW Set LLDP MIB API if FW > v4.40 */
12807         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12808             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12809             (pf->hw.aq.fw_maj_ver >= 5)))
12810                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12811
12812         /* Enable PTP L4 if FW > v6.0 */
12813         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12814             pf->hw.aq.fw_maj_ver >= 6)
12815                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12816
12817         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12818                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12819                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12820                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12821         }
12822
12823         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12824                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
12825                 /* IWARP needs one extra vector for CQP just like MISC.*/
12826                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12827         }
12828         /* Stopping FW LLDP engine is supported on XL710 and X722
12829          * starting from FW versions determined in i40e_init_adminq.
12830          * Stopping the FW LLDP engine is not supported on XL710
12831          * if NPAR is functioning so unset this hw flag in this case.
12832          */
12833         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12834             pf->hw.func_caps.npar_enable &&
12835             (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12836                 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12837
12838 #ifdef CONFIG_PCI_IOV
12839         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12840                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12841                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12842                 pf->num_req_vfs = min_t(int,
12843                                         pf->hw.func_caps.num_vfs,
12844                                         I40E_MAX_VF_COUNT);
12845         }
12846 #endif /* CONFIG_PCI_IOV */
12847         pf->eeprom_version = 0xDEAD;
12848         pf->lan_veb = I40E_NO_VEB;
12849         pf->lan_vsi = I40E_NO_VSI;
12850
12851         /* By default FW has this off for performance reasons */
12852         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12853
12854         /* set up queue assignment tracking */
12855         size = sizeof(struct i40e_lump_tracking)
12856                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12857         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12858         if (!pf->qp_pile) {
12859                 err = -ENOMEM;
12860                 goto sw_init_done;
12861         }
12862         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12863
12864         pf->tx_timeout_recovery_level = 1;
12865
12866         if (pf->hw.mac.type != I40E_MAC_X722 &&
12867             i40e_is_total_port_shutdown_enabled(pf)) {
12868                 /* Link down on close must be on when total port shutdown
12869                  * is enabled for a given port
12870                  */
12871                 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12872                               I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12873                 dev_info(&pf->pdev->dev,
12874                          "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12875         }
12876         mutex_init(&pf->switch_mutex);
12877
12878 sw_init_done:
12879         return err;
12880 }
12881
12882 /**
12883  * i40e_set_ntuple - set the ntuple feature flag and take action
12884  * @pf: board private structure to initialize
12885  * @features: the feature set that the stack is suggesting
12886  *
12887  * returns a bool to indicate if reset needs to happen
12888  **/
12889 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12890 {
12891         bool need_reset = false;
12892
12893         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12894          * the state changed, we need to reset.
12895          */
12896         if (features & NETIF_F_NTUPLE) {
12897                 /* Enable filters and mark for reset */
12898                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12899                         need_reset = true;
12900                 /* enable FD_SB only if there is MSI-X vector and no cloud
12901                  * filters exist
12902                  */
12903                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12904                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12905                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12906                 }
12907         } else {
12908                 /* turn off filters, mark for reset and clear SW filter list */
12909                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12910                         need_reset = true;
12911                         i40e_fdir_filter_exit(pf);
12912                 }
12913                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12914                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12915                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12916
12917                 /* reset fd counters */
12918                 pf->fd_add_err = 0;
12919                 pf->fd_atr_cnt = 0;
12920                 /* if ATR was auto disabled it can be re-enabled. */
12921                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12922                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12923                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12924                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12925         }
12926         return need_reset;
12927 }
12928
12929 /**
12930  * i40e_clear_rss_lut - clear the rx hash lookup table
12931  * @vsi: the VSI being configured
12932  **/
12933 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12934 {
12935         struct i40e_pf *pf = vsi->back;
12936         struct i40e_hw *hw = &pf->hw;
12937         u16 vf_id = vsi->vf_id;
12938         u8 i;
12939
12940         if (vsi->type == I40E_VSI_MAIN) {
12941                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12942                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12943         } else if (vsi->type == I40E_VSI_SRIOV) {
12944                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12945                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12946         } else {
12947                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12948         }
12949 }
12950
12951 /**
12952  * i40e_set_features - set the netdev feature flags
12953  * @netdev: ptr to the netdev being adjusted
12954  * @features: the feature set that the stack is suggesting
12955  * Note: expects to be called while under rtnl_lock()
12956  **/
12957 static int i40e_set_features(struct net_device *netdev,
12958                              netdev_features_t features)
12959 {
12960         struct i40e_netdev_priv *np = netdev_priv(netdev);
12961         struct i40e_vsi *vsi = np->vsi;
12962         struct i40e_pf *pf = vsi->back;
12963         bool need_reset;
12964
12965         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12966                 i40e_pf_config_rss(pf);
12967         else if (!(features & NETIF_F_RXHASH) &&
12968                  netdev->features & NETIF_F_RXHASH)
12969                 i40e_clear_rss_lut(vsi);
12970
12971         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12972                 i40e_vlan_stripping_enable(vsi);
12973         else
12974                 i40e_vlan_stripping_disable(vsi);
12975
12976         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12977                 dev_err(&pf->pdev->dev,
12978                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12979                 return -EINVAL;
12980         }
12981
12982         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12983                 i40e_del_all_macvlans(vsi);
12984
12985         need_reset = i40e_set_ntuple(pf, features);
12986
12987         if (need_reset)
12988                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12989
12990         return 0;
12991 }
12992
12993 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12994                                     unsigned int table, unsigned int idx,
12995                                     struct udp_tunnel_info *ti)
12996 {
12997         struct i40e_netdev_priv *np = netdev_priv(netdev);
12998         struct i40e_hw *hw = &np->vsi->back->hw;
12999         u8 type, filter_index;
13000         int ret;
13001
13002         type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
13003                                                    I40E_AQC_TUNNEL_TYPE_NGE;
13004
13005         ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
13006                                      NULL);
13007         if (ret) {
13008                 netdev_info(netdev, "add UDP port failed, err %d aq_err %s\n",
13009                             ret,
13010                             i40e_aq_str(hw, hw->aq.asq_last_status));
13011                 return -EIO;
13012         }
13013
13014         udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
13015         return 0;
13016 }
13017
13018 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
13019                                       unsigned int table, unsigned int idx,
13020                                       struct udp_tunnel_info *ti)
13021 {
13022         struct i40e_netdev_priv *np = netdev_priv(netdev);
13023         struct i40e_hw *hw = &np->vsi->back->hw;
13024         int ret;
13025
13026         ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13027         if (ret) {
13028                 netdev_info(netdev, "delete UDP port failed, err %d aq_err %s\n",
13029                             ret,
13030                             i40e_aq_str(hw, hw->aq.asq_last_status));
13031                 return -EIO;
13032         }
13033
13034         return 0;
13035 }
13036
13037 static int i40e_get_phys_port_id(struct net_device *netdev,
13038                                  struct netdev_phys_item_id *ppid)
13039 {
13040         struct i40e_netdev_priv *np = netdev_priv(netdev);
13041         struct i40e_pf *pf = np->vsi->back;
13042         struct i40e_hw *hw = &pf->hw;
13043
13044         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
13045                 return -EOPNOTSUPP;
13046
13047         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13048         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13049
13050         return 0;
13051 }
13052
13053 /**
13054  * i40e_ndo_fdb_add - add an entry to the hardware database
13055  * @ndm: the input from the stack
13056  * @tb: pointer to array of nladdr (unused)
13057  * @dev: the net device pointer
13058  * @addr: the MAC address entry being added
13059  * @vid: VLAN ID
13060  * @flags: instructions from stack about fdb operation
13061  * @extack: netlink extended ack, unused currently
13062  */
13063 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13064                             struct net_device *dev,
13065                             const unsigned char *addr, u16 vid,
13066                             u16 flags,
13067                             struct netlink_ext_ack *extack)
13068 {
13069         struct i40e_netdev_priv *np = netdev_priv(dev);
13070         struct i40e_pf *pf = np->vsi->back;
13071         int err = 0;
13072
13073         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
13074                 return -EOPNOTSUPP;
13075
13076         if (vid) {
13077                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13078                 return -EINVAL;
13079         }
13080
13081         /* Hardware does not support aging addresses so if a
13082          * ndm_state is given only allow permanent addresses
13083          */
13084         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13085                 netdev_info(dev, "FDB only supports static addresses\n");
13086                 return -EINVAL;
13087         }
13088
13089         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13090                 err = dev_uc_add_excl(dev, addr);
13091         else if (is_multicast_ether_addr(addr))
13092                 err = dev_mc_add_excl(dev, addr);
13093         else
13094                 err = -EINVAL;
13095
13096         /* Only return duplicate errors if NLM_F_EXCL is set */
13097         if (err == -EEXIST && !(flags & NLM_F_EXCL))
13098                 err = 0;
13099
13100         return err;
13101 }
13102
13103 /**
13104  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13105  * @dev: the netdev being configured
13106  * @nlh: RTNL message
13107  * @flags: bridge flags
13108  * @extack: netlink extended ack
13109  *
13110  * Inserts a new hardware bridge if not already created and
13111  * enables the bridging mode requested (VEB or VEPA). If the
13112  * hardware bridge has already been inserted and the request
13113  * is to change the mode then that requires a PF reset to
13114  * allow rebuild of the components with required hardware
13115  * bridge mode enabled.
13116  *
13117  * Note: expects to be called while under rtnl_lock()
13118  **/
13119 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13120                                    struct nlmsghdr *nlh,
13121                                    u16 flags,
13122                                    struct netlink_ext_ack *extack)
13123 {
13124         struct i40e_netdev_priv *np = netdev_priv(dev);
13125         struct i40e_vsi *vsi = np->vsi;
13126         struct i40e_pf *pf = vsi->back;
13127         struct i40e_veb *veb = NULL;
13128         struct nlattr *attr, *br_spec;
13129         int i, rem;
13130
13131         /* Only for PF VSI for now */
13132         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13133                 return -EOPNOTSUPP;
13134
13135         /* Find the HW bridge for PF VSI */
13136         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13137                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13138                         veb = pf->veb[i];
13139         }
13140
13141         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13142         if (!br_spec)
13143                 return -EINVAL;
13144
13145         nla_for_each_nested(attr, br_spec, rem) {
13146                 __u16 mode;
13147
13148                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
13149                         continue;
13150
13151                 mode = nla_get_u16(attr);
13152                 if ((mode != BRIDGE_MODE_VEPA) &&
13153                     (mode != BRIDGE_MODE_VEB))
13154                         return -EINVAL;
13155
13156                 /* Insert a new HW bridge */
13157                 if (!veb) {
13158                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13159                                              vsi->tc_config.enabled_tc);
13160                         if (veb) {
13161                                 veb->bridge_mode = mode;
13162                                 i40e_config_bridge_mode(veb);
13163                         } else {
13164                                 /* No Bridge HW offload available */
13165                                 return -ENOENT;
13166                         }
13167                         break;
13168                 } else if (mode != veb->bridge_mode) {
13169                         /* Existing HW bridge but different mode needs reset */
13170                         veb->bridge_mode = mode;
13171                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13172                         if (mode == BRIDGE_MODE_VEB)
13173                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
13174                         else
13175                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
13176                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13177                         break;
13178                 }
13179         }
13180
13181         return 0;
13182 }
13183
13184 /**
13185  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13186  * @skb: skb buff
13187  * @pid: process id
13188  * @seq: RTNL message seq #
13189  * @dev: the netdev being configured
13190  * @filter_mask: unused
13191  * @nlflags: netlink flags passed in
13192  *
13193  * Return the mode in which the hardware bridge is operating in
13194  * i.e VEB or VEPA.
13195  **/
13196 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13197                                    struct net_device *dev,
13198                                    u32 __always_unused filter_mask,
13199                                    int nlflags)
13200 {
13201         struct i40e_netdev_priv *np = netdev_priv(dev);
13202         struct i40e_vsi *vsi = np->vsi;
13203         struct i40e_pf *pf = vsi->back;
13204         struct i40e_veb *veb = NULL;
13205         int i;
13206
13207         /* Only for PF VSI for now */
13208         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13209                 return -EOPNOTSUPP;
13210
13211         /* Find the HW bridge for the PF VSI */
13212         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13213                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13214                         veb = pf->veb[i];
13215         }
13216
13217         if (!veb)
13218                 return 0;
13219
13220         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13221                                        0, 0, nlflags, filter_mask, NULL);
13222 }
13223
13224 /**
13225  * i40e_features_check - Validate encapsulated packet conforms to limits
13226  * @skb: skb buff
13227  * @dev: This physical port's netdev
13228  * @features: Offload features that the stack believes apply
13229  **/
13230 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13231                                              struct net_device *dev,
13232                                              netdev_features_t features)
13233 {
13234         size_t len;
13235
13236         /* No point in doing any of this if neither checksum nor GSO are
13237          * being requested for this frame.  We can rule out both by just
13238          * checking for CHECKSUM_PARTIAL
13239          */
13240         if (skb->ip_summed != CHECKSUM_PARTIAL)
13241                 return features;
13242
13243         /* We cannot support GSO if the MSS is going to be less than
13244          * 64 bytes.  If it is then we need to drop support for GSO.
13245          */
13246         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13247                 features &= ~NETIF_F_GSO_MASK;
13248
13249         /* MACLEN can support at most 63 words */
13250         len = skb_network_header(skb) - skb->data;
13251         if (len & ~(63 * 2))
13252                 goto out_err;
13253
13254         /* IPLEN and EIPLEN can support at most 127 dwords */
13255         len = skb_transport_header(skb) - skb_network_header(skb);
13256         if (len & ~(127 * 4))
13257                 goto out_err;
13258
13259         if (skb->encapsulation) {
13260                 /* L4TUNLEN can support 127 words */
13261                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
13262                 if (len & ~(127 * 2))
13263                         goto out_err;
13264
13265                 /* IPLEN can support at most 127 dwords */
13266                 len = skb_inner_transport_header(skb) -
13267                       skb_inner_network_header(skb);
13268                 if (len & ~(127 * 4))
13269                         goto out_err;
13270         }
13271
13272         /* No need to validate L4LEN as TCP is the only protocol with a
13273          * a flexible value and we support all possible values supported
13274          * by TCP, which is at most 15 dwords
13275          */
13276
13277         return features;
13278 out_err:
13279         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13280 }
13281
13282 /**
13283  * i40e_xdp_setup - add/remove an XDP program
13284  * @vsi: VSI to changed
13285  * @prog: XDP program
13286  * @extack: netlink extended ack
13287  **/
13288 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13289                           struct netlink_ext_ack *extack)
13290 {
13291         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
13292         struct i40e_pf *pf = vsi->back;
13293         struct bpf_prog *old_prog;
13294         bool need_reset;
13295         int i;
13296
13297         /* Don't allow frames that span over multiple buffers */
13298         if (frame_size > i40e_calculate_vsi_rx_buf_len(vsi)) {
13299                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
13300                 return -EINVAL;
13301         }
13302
13303         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
13304         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13305
13306         if (need_reset)
13307                 i40e_prep_for_reset(pf);
13308
13309         /* VSI shall be deleted in a moment, just return EINVAL */
13310         if (test_bit(__I40E_IN_REMOVE, pf->state))
13311                 return -EINVAL;
13312
13313         old_prog = xchg(&vsi->xdp_prog, prog);
13314
13315         if (need_reset) {
13316                 if (!prog)
13317                         /* Wait until ndo_xsk_wakeup completes. */
13318                         synchronize_rcu();
13319                 i40e_reset_and_rebuild(pf, true, true);
13320         }
13321
13322         if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13323                 if (i40e_realloc_rx_bi_zc(vsi, true))
13324                         return -ENOMEM;
13325         } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13326                 if (i40e_realloc_rx_bi_zc(vsi, false))
13327                         return -ENOMEM;
13328         }
13329
13330         for (i = 0; i < vsi->num_queue_pairs; i++)
13331                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13332
13333         if (old_prog)
13334                 bpf_prog_put(old_prog);
13335
13336         /* Kick start the NAPI context if there is an AF_XDP socket open
13337          * on that queue id. This so that receiving will start.
13338          */
13339         if (need_reset && prog)
13340                 for (i = 0; i < vsi->num_queue_pairs; i++)
13341                         if (vsi->xdp_rings[i]->xsk_pool)
13342                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
13343                                                       XDP_WAKEUP_RX);
13344
13345         return 0;
13346 }
13347
13348 /**
13349  * i40e_enter_busy_conf - Enters busy config state
13350  * @vsi: vsi
13351  *
13352  * Returns 0 on success, <0 for failure.
13353  **/
13354 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13355 {
13356         struct i40e_pf *pf = vsi->back;
13357         int timeout = 50;
13358
13359         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13360                 timeout--;
13361                 if (!timeout)
13362                         return -EBUSY;
13363                 usleep_range(1000, 2000);
13364         }
13365
13366         return 0;
13367 }
13368
13369 /**
13370  * i40e_exit_busy_conf - Exits busy config state
13371  * @vsi: vsi
13372  **/
13373 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13374 {
13375         struct i40e_pf *pf = vsi->back;
13376
13377         clear_bit(__I40E_CONFIG_BUSY, pf->state);
13378 }
13379
13380 /**
13381  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13382  * @vsi: vsi
13383  * @queue_pair: queue pair
13384  **/
13385 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13386 {
13387         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13388                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13389         memset(&vsi->tx_rings[queue_pair]->stats, 0,
13390                sizeof(vsi->tx_rings[queue_pair]->stats));
13391         if (i40e_enabled_xdp_vsi(vsi)) {
13392                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13393                        sizeof(vsi->xdp_rings[queue_pair]->stats));
13394         }
13395 }
13396
13397 /**
13398  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13399  * @vsi: vsi
13400  * @queue_pair: queue pair
13401  **/
13402 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13403 {
13404         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13405         if (i40e_enabled_xdp_vsi(vsi)) {
13406                 /* Make sure that in-progress ndo_xdp_xmit calls are
13407                  * completed.
13408                  */
13409                 synchronize_rcu();
13410                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13411         }
13412         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13413 }
13414
13415 /**
13416  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13417  * @vsi: vsi
13418  * @queue_pair: queue pair
13419  * @enable: true for enable, false for disable
13420  **/
13421 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13422                                         bool enable)
13423 {
13424         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13425         struct i40e_q_vector *q_vector = rxr->q_vector;
13426
13427         if (!vsi->netdev)
13428                 return;
13429
13430         /* All rings in a qp belong to the same qvector. */
13431         if (q_vector->rx.ring || q_vector->tx.ring) {
13432                 if (enable)
13433                         napi_enable(&q_vector->napi);
13434                 else
13435                         napi_disable(&q_vector->napi);
13436         }
13437 }
13438
13439 /**
13440  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13441  * @vsi: vsi
13442  * @queue_pair: queue pair
13443  * @enable: true for enable, false for disable
13444  *
13445  * Returns 0 on success, <0 on failure.
13446  **/
13447 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13448                                         bool enable)
13449 {
13450         struct i40e_pf *pf = vsi->back;
13451         int pf_q, ret = 0;
13452
13453         pf_q = vsi->base_queue + queue_pair;
13454         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13455                                      false /*is xdp*/, enable);
13456         if (ret) {
13457                 dev_info(&pf->pdev->dev,
13458                          "VSI seid %d Tx ring %d %sable timeout\n",
13459                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13460                 return ret;
13461         }
13462
13463         i40e_control_rx_q(pf, pf_q, enable);
13464         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13465         if (ret) {
13466                 dev_info(&pf->pdev->dev,
13467                          "VSI seid %d Rx ring %d %sable timeout\n",
13468                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13469                 return ret;
13470         }
13471
13472         /* Due to HW errata, on Rx disable only, the register can
13473          * indicate done before it really is. Needs 50ms to be sure
13474          */
13475         if (!enable)
13476                 mdelay(50);
13477
13478         if (!i40e_enabled_xdp_vsi(vsi))
13479                 return ret;
13480
13481         ret = i40e_control_wait_tx_q(vsi->seid, pf,
13482                                      pf_q + vsi->alloc_queue_pairs,
13483                                      true /*is xdp*/, enable);
13484         if (ret) {
13485                 dev_info(&pf->pdev->dev,
13486                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
13487                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13488         }
13489
13490         return ret;
13491 }
13492
13493 /**
13494  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13495  * @vsi: vsi
13496  * @queue_pair: queue_pair
13497  **/
13498 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13499 {
13500         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13501         struct i40e_pf *pf = vsi->back;
13502         struct i40e_hw *hw = &pf->hw;
13503
13504         /* All rings in a qp belong to the same qvector. */
13505         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
13506                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13507         else
13508                 i40e_irq_dynamic_enable_icr0(pf);
13509
13510         i40e_flush(hw);
13511 }
13512
13513 /**
13514  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13515  * @vsi: vsi
13516  * @queue_pair: queue_pair
13517  **/
13518 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13519 {
13520         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13521         struct i40e_pf *pf = vsi->back;
13522         struct i40e_hw *hw = &pf->hw;
13523
13524         /* For simplicity, instead of removing the qp interrupt causes
13525          * from the interrupt linked list, we simply disable the interrupt, and
13526          * leave the list intact.
13527          *
13528          * All rings in a qp belong to the same qvector.
13529          */
13530         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13531                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13532
13533                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13534                 i40e_flush(hw);
13535                 synchronize_irq(pf->msix_entries[intpf].vector);
13536         } else {
13537                 /* Legacy and MSI mode - this stops all interrupt handling */
13538                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13539                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13540                 i40e_flush(hw);
13541                 synchronize_irq(pf->pdev->irq);
13542         }
13543 }
13544
13545 /**
13546  * i40e_queue_pair_disable - Disables a queue pair
13547  * @vsi: vsi
13548  * @queue_pair: queue pair
13549  *
13550  * Returns 0 on success, <0 on failure.
13551  **/
13552 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13553 {
13554         int err;
13555
13556         err = i40e_enter_busy_conf(vsi);
13557         if (err)
13558                 return err;
13559
13560         i40e_queue_pair_disable_irq(vsi, queue_pair);
13561         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13562         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13563         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13564         i40e_queue_pair_clean_rings(vsi, queue_pair);
13565         i40e_queue_pair_reset_stats(vsi, queue_pair);
13566
13567         return err;
13568 }
13569
13570 /**
13571  * i40e_queue_pair_enable - Enables a queue pair
13572  * @vsi: vsi
13573  * @queue_pair: queue pair
13574  *
13575  * Returns 0 on success, <0 on failure.
13576  **/
13577 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13578 {
13579         int err;
13580
13581         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13582         if (err)
13583                 return err;
13584
13585         if (i40e_enabled_xdp_vsi(vsi)) {
13586                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13587                 if (err)
13588                         return err;
13589         }
13590
13591         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13592         if (err)
13593                 return err;
13594
13595         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13596         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13597         i40e_queue_pair_enable_irq(vsi, queue_pair);
13598
13599         i40e_exit_busy_conf(vsi);
13600
13601         return err;
13602 }
13603
13604 /**
13605  * i40e_xdp - implements ndo_bpf for i40e
13606  * @dev: netdevice
13607  * @xdp: XDP command
13608  **/
13609 static int i40e_xdp(struct net_device *dev,
13610                     struct netdev_bpf *xdp)
13611 {
13612         struct i40e_netdev_priv *np = netdev_priv(dev);
13613         struct i40e_vsi *vsi = np->vsi;
13614
13615         if (vsi->type != I40E_VSI_MAIN)
13616                 return -EINVAL;
13617
13618         switch (xdp->command) {
13619         case XDP_SETUP_PROG:
13620                 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13621         case XDP_SETUP_XSK_POOL:
13622                 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13623                                            xdp->xsk.queue_id);
13624         default:
13625                 return -EINVAL;
13626         }
13627 }
13628
13629 static const struct net_device_ops i40e_netdev_ops = {
13630         .ndo_open               = i40e_open,
13631         .ndo_stop               = i40e_close,
13632         .ndo_start_xmit         = i40e_lan_xmit_frame,
13633         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
13634         .ndo_set_rx_mode        = i40e_set_rx_mode,
13635         .ndo_validate_addr      = eth_validate_addr,
13636         .ndo_set_mac_address    = i40e_set_mac,
13637         .ndo_change_mtu         = i40e_change_mtu,
13638         .ndo_eth_ioctl          = i40e_ioctl,
13639         .ndo_tx_timeout         = i40e_tx_timeout,
13640         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
13641         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
13642 #ifdef CONFIG_NET_POLL_CONTROLLER
13643         .ndo_poll_controller    = i40e_netpoll,
13644 #endif
13645         .ndo_setup_tc           = __i40e_setup_tc,
13646         .ndo_select_queue       = i40e_lan_select_queue,
13647         .ndo_set_features       = i40e_set_features,
13648         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
13649         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
13650         .ndo_get_vf_stats       = i40e_get_vf_stats,
13651         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
13652         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
13653         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
13654         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
13655         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
13656         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
13657         .ndo_fdb_add            = i40e_ndo_fdb_add,
13658         .ndo_features_check     = i40e_features_check,
13659         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
13660         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
13661         .ndo_bpf                = i40e_xdp,
13662         .ndo_xdp_xmit           = i40e_xdp_xmit,
13663         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
13664         .ndo_dfwd_add_station   = i40e_fwd_add,
13665         .ndo_dfwd_del_station   = i40e_fwd_del,
13666 };
13667
13668 /**
13669  * i40e_config_netdev - Setup the netdev flags
13670  * @vsi: the VSI being configured
13671  *
13672  * Returns 0 on success, negative value on failure
13673  **/
13674 static int i40e_config_netdev(struct i40e_vsi *vsi)
13675 {
13676         struct i40e_pf *pf = vsi->back;
13677         struct i40e_hw *hw = &pf->hw;
13678         struct i40e_netdev_priv *np;
13679         struct net_device *netdev;
13680         u8 broadcast[ETH_ALEN];
13681         u8 mac_addr[ETH_ALEN];
13682         int etherdev_size;
13683         netdev_features_t hw_enc_features;
13684         netdev_features_t hw_features;
13685
13686         etherdev_size = sizeof(struct i40e_netdev_priv);
13687         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13688         if (!netdev)
13689                 return -ENOMEM;
13690
13691         vsi->netdev = netdev;
13692         np = netdev_priv(netdev);
13693         np->vsi = vsi;
13694
13695         hw_enc_features = NETIF_F_SG                    |
13696                           NETIF_F_IP_CSUM               |
13697                           NETIF_F_IPV6_CSUM             |
13698                           NETIF_F_HIGHDMA               |
13699                           NETIF_F_SOFT_FEATURES         |
13700                           NETIF_F_TSO                   |
13701                           NETIF_F_TSO_ECN               |
13702                           NETIF_F_TSO6                  |
13703                           NETIF_F_GSO_GRE               |
13704                           NETIF_F_GSO_GRE_CSUM          |
13705                           NETIF_F_GSO_PARTIAL           |
13706                           NETIF_F_GSO_IPXIP4            |
13707                           NETIF_F_GSO_IPXIP6            |
13708                           NETIF_F_GSO_UDP_TUNNEL        |
13709                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
13710                           NETIF_F_GSO_UDP_L4            |
13711                           NETIF_F_SCTP_CRC              |
13712                           NETIF_F_RXHASH                |
13713                           NETIF_F_RXCSUM                |
13714                           0;
13715
13716         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13717                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13718
13719         netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13720
13721         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13722
13723         netdev->hw_enc_features |= hw_enc_features;
13724
13725         /* record features VLANs can make use of */
13726         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13727
13728         /* enable macvlan offloads */
13729         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13730
13731         hw_features = hw_enc_features           |
13732                       NETIF_F_HW_VLAN_CTAG_TX   |
13733                       NETIF_F_HW_VLAN_CTAG_RX;
13734
13735         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13736                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13737
13738         netdev->hw_features |= hw_features;
13739
13740         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13741         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13742
13743         if (vsi->type == I40E_VSI_MAIN) {
13744                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13745                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13746                 /* The following steps are necessary for two reasons. First,
13747                  * some older NVM configurations load a default MAC-VLAN
13748                  * filter that will accept any tagged packet, and we want to
13749                  * replace this with a normal filter. Additionally, it is
13750                  * possible our MAC address was provided by the platform using
13751                  * Open Firmware or similar.
13752                  *
13753                  * Thus, we need to remove the default filter and install one
13754                  * specific to the MAC address.
13755                  */
13756                 i40e_rm_default_mac_filter(vsi, mac_addr);
13757                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13758                 i40e_add_mac_filter(vsi, mac_addr);
13759                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13760         } else {
13761                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13762                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13763                  * the end, which is 4 bytes long, so force truncation of the
13764                  * original name by IFNAMSIZ - 4
13765                  */
13766                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13767                          IFNAMSIZ - 4,
13768                          pf->vsi[pf->lan_vsi]->netdev->name);
13769                 eth_random_addr(mac_addr);
13770
13771                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13772                 i40e_add_mac_filter(vsi, mac_addr);
13773                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13774         }
13775
13776         /* Add the broadcast filter so that we initially will receive
13777          * broadcast packets. Note that when a new VLAN is first added the
13778          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13779          * specific filters as part of transitioning into "vlan" operation.
13780          * When more VLANs are added, the driver will copy each existing MAC
13781          * filter and add it for the new VLAN.
13782          *
13783          * Broadcast filters are handled specially by
13784          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13785          * promiscuous bit instead of adding this directly as a MAC/VLAN
13786          * filter. The subtask will update the correct broadcast promiscuous
13787          * bits as VLANs become active or inactive.
13788          */
13789         eth_broadcast_addr(broadcast);
13790         spin_lock_bh(&vsi->mac_filter_hash_lock);
13791         i40e_add_mac_filter(vsi, broadcast);
13792         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13793
13794         eth_hw_addr_set(netdev, mac_addr);
13795         ether_addr_copy(netdev->perm_addr, mac_addr);
13796
13797         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13798         netdev->neigh_priv_len = sizeof(u32) * 4;
13799
13800         netdev->priv_flags |= IFF_UNICAST_FLT;
13801         netdev->priv_flags |= IFF_SUPP_NOFCS;
13802         /* Setup netdev TC information */
13803         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13804
13805         netdev->netdev_ops = &i40e_netdev_ops;
13806         netdev->watchdog_timeo = 5 * HZ;
13807         i40e_set_ethtool_ops(netdev);
13808
13809         /* MTU range: 68 - 9706 */
13810         netdev->min_mtu = ETH_MIN_MTU;
13811         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13812
13813         return 0;
13814 }
13815
13816 /**
13817  * i40e_vsi_delete - Delete a VSI from the switch
13818  * @vsi: the VSI being removed
13819  *
13820  * Returns 0 on success, negative value on failure
13821  **/
13822 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13823 {
13824         /* remove default VSI is not allowed */
13825         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13826                 return;
13827
13828         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13829 }
13830
13831 /**
13832  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13833  * @vsi: the VSI being queried
13834  *
13835  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13836  **/
13837 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13838 {
13839         struct i40e_veb *veb;
13840         struct i40e_pf *pf = vsi->back;
13841
13842         /* Uplink is not a bridge so default to VEB */
13843         if (vsi->veb_idx >= I40E_MAX_VEB)
13844                 return 1;
13845
13846         veb = pf->veb[vsi->veb_idx];
13847         if (!veb) {
13848                 dev_info(&pf->pdev->dev,
13849                          "There is no veb associated with the bridge\n");
13850                 return -ENOENT;
13851         }
13852
13853         /* Uplink is a bridge in VEPA mode */
13854         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13855                 return 0;
13856         } else {
13857                 /* Uplink is a bridge in VEB mode */
13858                 return 1;
13859         }
13860
13861         /* VEPA is now default bridge, so return 0 */
13862         return 0;
13863 }
13864
13865 /**
13866  * i40e_add_vsi - Add a VSI to the switch
13867  * @vsi: the VSI being configured
13868  *
13869  * This initializes a VSI context depending on the VSI type to be added and
13870  * passes it down to the add_vsi aq command.
13871  **/
13872 static int i40e_add_vsi(struct i40e_vsi *vsi)
13873 {
13874         int ret = -ENODEV;
13875         struct i40e_pf *pf = vsi->back;
13876         struct i40e_hw *hw = &pf->hw;
13877         struct i40e_vsi_context ctxt;
13878         struct i40e_mac_filter *f;
13879         struct hlist_node *h;
13880         int bkt;
13881
13882         u8 enabled_tc = 0x1; /* TC0 enabled */
13883         int f_count = 0;
13884
13885         memset(&ctxt, 0, sizeof(ctxt));
13886         switch (vsi->type) {
13887         case I40E_VSI_MAIN:
13888                 /* The PF's main VSI is already setup as part of the
13889                  * device initialization, so we'll not bother with
13890                  * the add_vsi call, but we will retrieve the current
13891                  * VSI context.
13892                  */
13893                 ctxt.seid = pf->main_vsi_seid;
13894                 ctxt.pf_num = pf->hw.pf_id;
13895                 ctxt.vf_num = 0;
13896                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13897                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13898                 if (ret) {
13899                         dev_info(&pf->pdev->dev,
13900                                  "couldn't get PF vsi config, err %d aq_err %s\n",
13901                                  ret,
13902                                  i40e_aq_str(&pf->hw,
13903                                              pf->hw.aq.asq_last_status));
13904                         return -ENOENT;
13905                 }
13906                 vsi->info = ctxt.info;
13907                 vsi->info.valid_sections = 0;
13908
13909                 vsi->seid = ctxt.seid;
13910                 vsi->id = ctxt.vsi_number;
13911
13912                 enabled_tc = i40e_pf_get_tc_map(pf);
13913
13914                 /* Source pruning is enabled by default, so the flag is
13915                  * negative logic - if it's set, we need to fiddle with
13916                  * the VSI to disable source pruning.
13917                  */
13918                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13919                         memset(&ctxt, 0, sizeof(ctxt));
13920                         ctxt.seid = pf->main_vsi_seid;
13921                         ctxt.pf_num = pf->hw.pf_id;
13922                         ctxt.vf_num = 0;
13923                         ctxt.info.valid_sections |=
13924                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13925                         ctxt.info.switch_id =
13926                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13927                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13928                         if (ret) {
13929                                 dev_info(&pf->pdev->dev,
13930                                          "update vsi failed, err %d aq_err %s\n",
13931                                          ret,
13932                                          i40e_aq_str(&pf->hw,
13933                                                      pf->hw.aq.asq_last_status));
13934                                 ret = -ENOENT;
13935                                 goto err;
13936                         }
13937                 }
13938
13939                 /* MFP mode setup queue map and update VSI */
13940                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13941                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13942                         memset(&ctxt, 0, sizeof(ctxt));
13943                         ctxt.seid = pf->main_vsi_seid;
13944                         ctxt.pf_num = pf->hw.pf_id;
13945                         ctxt.vf_num = 0;
13946                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13947                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13948                         if (ret) {
13949                                 dev_info(&pf->pdev->dev,
13950                                          "update vsi failed, err %d aq_err %s\n",
13951                                          ret,
13952                                          i40e_aq_str(&pf->hw,
13953                                                     pf->hw.aq.asq_last_status));
13954                                 ret = -ENOENT;
13955                                 goto err;
13956                         }
13957                         /* update the local VSI info queue map */
13958                         i40e_vsi_update_queue_map(vsi, &ctxt);
13959                         vsi->info.valid_sections = 0;
13960                 } else {
13961                         /* Default/Main VSI is only enabled for TC0
13962                          * reconfigure it to enable all TCs that are
13963                          * available on the port in SFP mode.
13964                          * For MFP case the iSCSI PF would use this
13965                          * flow to enable LAN+iSCSI TC.
13966                          */
13967                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13968                         if (ret) {
13969                                 /* Single TC condition is not fatal,
13970                                  * message and continue
13971                                  */
13972                                 dev_info(&pf->pdev->dev,
13973                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d aq_err %s\n",
13974                                          enabled_tc,
13975                                          ret,
13976                                          i40e_aq_str(&pf->hw,
13977                                                     pf->hw.aq.asq_last_status));
13978                         }
13979                 }
13980                 break;
13981
13982         case I40E_VSI_FDIR:
13983                 ctxt.pf_num = hw->pf_id;
13984                 ctxt.vf_num = 0;
13985                 ctxt.uplink_seid = vsi->uplink_seid;
13986                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13987                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13988                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13989                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13990                         ctxt.info.valid_sections |=
13991                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13992                         ctxt.info.switch_id =
13993                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13994                 }
13995                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13996                 break;
13997
13998         case I40E_VSI_VMDQ2:
13999                 ctxt.pf_num = hw->pf_id;
14000                 ctxt.vf_num = 0;
14001                 ctxt.uplink_seid = vsi->uplink_seid;
14002                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14003                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14004
14005                 /* This VSI is connected to VEB so the switch_id
14006                  * should be set to zero by default.
14007                  */
14008                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14009                         ctxt.info.valid_sections |=
14010                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14011                         ctxt.info.switch_id =
14012                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14013                 }
14014
14015                 /* Setup the VSI tx/rx queue map for TC0 only for now */
14016                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14017                 break;
14018
14019         case I40E_VSI_SRIOV:
14020                 ctxt.pf_num = hw->pf_id;
14021                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14022                 ctxt.uplink_seid = vsi->uplink_seid;
14023                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14024                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14025
14026                 /* This VSI is connected to VEB so the switch_id
14027                  * should be set to zero by default.
14028                  */
14029                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14030                         ctxt.info.valid_sections |=
14031                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14032                         ctxt.info.switch_id =
14033                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14034                 }
14035
14036                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
14037                         ctxt.info.valid_sections |=
14038                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14039                         ctxt.info.queueing_opt_flags |=
14040                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14041                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14042                 }
14043
14044                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14045                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14046                 if (pf->vf[vsi->vf_id].spoofchk) {
14047                         ctxt.info.valid_sections |=
14048                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14049                         ctxt.info.sec_flags |=
14050                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14051                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14052                 }
14053                 /* Setup the VSI tx/rx queue map for TC0 only for now */
14054                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14055                 break;
14056
14057         case I40E_VSI_IWARP:
14058                 /* send down message to iWARP */
14059                 break;
14060
14061         default:
14062                 return -ENODEV;
14063         }
14064
14065         if (vsi->type != I40E_VSI_MAIN) {
14066                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14067                 if (ret) {
14068                         dev_info(&vsi->back->pdev->dev,
14069                                  "add vsi failed, err %d aq_err %s\n",
14070                                  ret,
14071                                  i40e_aq_str(&pf->hw,
14072                                              pf->hw.aq.asq_last_status));
14073                         ret = -ENOENT;
14074                         goto err;
14075                 }
14076                 vsi->info = ctxt.info;
14077                 vsi->info.valid_sections = 0;
14078                 vsi->seid = ctxt.seid;
14079                 vsi->id = ctxt.vsi_number;
14080         }
14081
14082         spin_lock_bh(&vsi->mac_filter_hash_lock);
14083         vsi->active_filters = 0;
14084         /* If macvlan filters already exist, force them to get loaded */
14085         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14086                 f->state = I40E_FILTER_NEW;
14087                 f_count++;
14088         }
14089         spin_unlock_bh(&vsi->mac_filter_hash_lock);
14090         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14091
14092         if (f_count) {
14093                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14094                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14095         }
14096
14097         /* Update VSI BW information */
14098         ret = i40e_vsi_get_bw_info(vsi);
14099         if (ret) {
14100                 dev_info(&pf->pdev->dev,
14101                          "couldn't get vsi bw info, err %d aq_err %s\n",
14102                          ret,
14103                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14104                 /* VSI is already added so not tearing that up */
14105                 ret = 0;
14106         }
14107
14108 err:
14109         return ret;
14110 }
14111
14112 /**
14113  * i40e_vsi_release - Delete a VSI and free its resources
14114  * @vsi: the VSI being removed
14115  *
14116  * Returns 0 on success or < 0 on error
14117  **/
14118 int i40e_vsi_release(struct i40e_vsi *vsi)
14119 {
14120         struct i40e_mac_filter *f;
14121         struct hlist_node *h;
14122         struct i40e_veb *veb = NULL;
14123         struct i40e_pf *pf;
14124         u16 uplink_seid;
14125         int i, n, bkt;
14126
14127         pf = vsi->back;
14128
14129         /* release of a VEB-owner or last VSI is not allowed */
14130         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14131                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14132                          vsi->seid, vsi->uplink_seid);
14133                 return -ENODEV;
14134         }
14135         if (vsi == pf->vsi[pf->lan_vsi] &&
14136             !test_bit(__I40E_DOWN, pf->state)) {
14137                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14138                 return -ENODEV;
14139         }
14140         set_bit(__I40E_VSI_RELEASING, vsi->state);
14141         uplink_seid = vsi->uplink_seid;
14142         if (vsi->type != I40E_VSI_SRIOV) {
14143                 if (vsi->netdev_registered) {
14144                         vsi->netdev_registered = false;
14145                         if (vsi->netdev) {
14146                                 /* results in a call to i40e_close() */
14147                                 unregister_netdev(vsi->netdev);
14148                         }
14149                 } else {
14150                         i40e_vsi_close(vsi);
14151                 }
14152                 i40e_vsi_disable_irq(vsi);
14153         }
14154
14155         spin_lock_bh(&vsi->mac_filter_hash_lock);
14156
14157         /* clear the sync flag on all filters */
14158         if (vsi->netdev) {
14159                 __dev_uc_unsync(vsi->netdev, NULL);
14160                 __dev_mc_unsync(vsi->netdev, NULL);
14161         }
14162
14163         /* make sure any remaining filters are marked for deletion */
14164         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14165                 __i40e_del_filter(vsi, f);
14166
14167         spin_unlock_bh(&vsi->mac_filter_hash_lock);
14168
14169         i40e_sync_vsi_filters(vsi);
14170
14171         i40e_vsi_delete(vsi);
14172         i40e_vsi_free_q_vectors(vsi);
14173         if (vsi->netdev) {
14174                 free_netdev(vsi->netdev);
14175                 vsi->netdev = NULL;
14176         }
14177         i40e_vsi_clear_rings(vsi);
14178         i40e_vsi_clear(vsi);
14179
14180         /* If this was the last thing on the VEB, except for the
14181          * controlling VSI, remove the VEB, which puts the controlling
14182          * VSI onto the next level down in the switch.
14183          *
14184          * Well, okay, there's one more exception here: don't remove
14185          * the orphan VEBs yet.  We'll wait for an explicit remove request
14186          * from up the network stack.
14187          */
14188         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
14189                 if (pf->vsi[i] &&
14190                     pf->vsi[i]->uplink_seid == uplink_seid &&
14191                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14192                         n++;      /* count the VSIs */
14193                 }
14194         }
14195         for (i = 0; i < I40E_MAX_VEB; i++) {
14196                 if (!pf->veb[i])
14197                         continue;
14198                 if (pf->veb[i]->uplink_seid == uplink_seid)
14199                         n++;     /* count the VEBs */
14200                 if (pf->veb[i]->seid == uplink_seid)
14201                         veb = pf->veb[i];
14202         }
14203         if (n == 0 && veb && veb->uplink_seid != 0)
14204                 i40e_veb_release(veb);
14205
14206         return 0;
14207 }
14208
14209 /**
14210  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14211  * @vsi: ptr to the VSI
14212  *
14213  * This should only be called after i40e_vsi_mem_alloc() which allocates the
14214  * corresponding SW VSI structure and initializes num_queue_pairs for the
14215  * newly allocated VSI.
14216  *
14217  * Returns 0 on success or negative on failure
14218  **/
14219 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14220 {
14221         int ret = -ENOENT;
14222         struct i40e_pf *pf = vsi->back;
14223
14224         if (vsi->q_vectors[0]) {
14225                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14226                          vsi->seid);
14227                 return -EEXIST;
14228         }
14229
14230         if (vsi->base_vector) {
14231                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14232                          vsi->seid, vsi->base_vector);
14233                 return -EEXIST;
14234         }
14235
14236         ret = i40e_vsi_alloc_q_vectors(vsi);
14237         if (ret) {
14238                 dev_info(&pf->pdev->dev,
14239                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14240                          vsi->num_q_vectors, vsi->seid, ret);
14241                 vsi->num_q_vectors = 0;
14242                 goto vector_setup_out;
14243         }
14244
14245         /* In Legacy mode, we do not have to get any other vector since we
14246          * piggyback on the misc/ICR0 for queue interrupts.
14247         */
14248         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
14249                 return ret;
14250         if (vsi->num_q_vectors)
14251                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14252                                                  vsi->num_q_vectors, vsi->idx);
14253         if (vsi->base_vector < 0) {
14254                 dev_info(&pf->pdev->dev,
14255                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14256                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14257                 i40e_vsi_free_q_vectors(vsi);
14258                 ret = -ENOENT;
14259                 goto vector_setup_out;
14260         }
14261
14262 vector_setup_out:
14263         return ret;
14264 }
14265
14266 /**
14267  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14268  * @vsi: pointer to the vsi.
14269  *
14270  * This re-allocates a vsi's queue resources.
14271  *
14272  * Returns pointer to the successfully allocated and configured VSI sw struct
14273  * on success, otherwise returns NULL on failure.
14274  **/
14275 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14276 {
14277         u16 alloc_queue_pairs;
14278         struct i40e_pf *pf;
14279         u8 enabled_tc;
14280         int ret;
14281
14282         if (!vsi)
14283                 return NULL;
14284
14285         pf = vsi->back;
14286
14287         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14288         i40e_vsi_clear_rings(vsi);
14289
14290         i40e_vsi_free_arrays(vsi, false);
14291         i40e_set_num_rings_in_vsi(vsi);
14292         ret = i40e_vsi_alloc_arrays(vsi, false);
14293         if (ret)
14294                 goto err_vsi;
14295
14296         alloc_queue_pairs = vsi->alloc_queue_pairs *
14297                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14298
14299         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14300         if (ret < 0) {
14301                 dev_info(&pf->pdev->dev,
14302                          "failed to get tracking for %d queues for VSI %d err %d\n",
14303                          alloc_queue_pairs, vsi->seid, ret);
14304                 goto err_vsi;
14305         }
14306         vsi->base_queue = ret;
14307
14308         /* Update the FW view of the VSI. Force a reset of TC and queue
14309          * layout configurations.
14310          */
14311         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14312         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14313         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14314         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14315         if (vsi->type == I40E_VSI_MAIN)
14316                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14317
14318         /* assign it some queues */
14319         ret = i40e_alloc_rings(vsi);
14320         if (ret)
14321                 goto err_rings;
14322
14323         /* map all of the rings to the q_vectors */
14324         i40e_vsi_map_rings_to_vectors(vsi);
14325         return vsi;
14326
14327 err_rings:
14328         i40e_vsi_free_q_vectors(vsi);
14329         if (vsi->netdev_registered) {
14330                 vsi->netdev_registered = false;
14331                 unregister_netdev(vsi->netdev);
14332                 free_netdev(vsi->netdev);
14333                 vsi->netdev = NULL;
14334         }
14335         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14336 err_vsi:
14337         i40e_vsi_clear(vsi);
14338         return NULL;
14339 }
14340
14341 /**
14342  * i40e_vsi_setup - Set up a VSI by a given type
14343  * @pf: board private structure
14344  * @type: VSI type
14345  * @uplink_seid: the switch element to link to
14346  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14347  *
14348  * This allocates the sw VSI structure and its queue resources, then add a VSI
14349  * to the identified VEB.
14350  *
14351  * Returns pointer to the successfully allocated and configure VSI sw struct on
14352  * success, otherwise returns NULL on failure.
14353  **/
14354 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14355                                 u16 uplink_seid, u32 param1)
14356 {
14357         struct i40e_vsi *vsi = NULL;
14358         struct i40e_veb *veb = NULL;
14359         u16 alloc_queue_pairs;
14360         int ret, i;
14361         int v_idx;
14362
14363         /* The requested uplink_seid must be either
14364          *     - the PF's port seid
14365          *              no VEB is needed because this is the PF
14366          *              or this is a Flow Director special case VSI
14367          *     - seid of an existing VEB
14368          *     - seid of a VSI that owns an existing VEB
14369          *     - seid of a VSI that doesn't own a VEB
14370          *              a new VEB is created and the VSI becomes the owner
14371          *     - seid of the PF VSI, which is what creates the first VEB
14372          *              this is a special case of the previous
14373          *
14374          * Find which uplink_seid we were given and create a new VEB if needed
14375          */
14376         for (i = 0; i < I40E_MAX_VEB; i++) {
14377                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
14378                         veb = pf->veb[i];
14379                         break;
14380                 }
14381         }
14382
14383         if (!veb && uplink_seid != pf->mac_seid) {
14384
14385                 for (i = 0; i < pf->num_alloc_vsi; i++) {
14386                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
14387                                 vsi = pf->vsi[i];
14388                                 break;
14389                         }
14390                 }
14391                 if (!vsi) {
14392                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14393                                  uplink_seid);
14394                         return NULL;
14395                 }
14396
14397                 if (vsi->uplink_seid == pf->mac_seid)
14398                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14399                                              vsi->tc_config.enabled_tc);
14400                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14401                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14402                                              vsi->tc_config.enabled_tc);
14403                 if (veb) {
14404                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14405                                 dev_info(&vsi->back->pdev->dev,
14406                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
14407                                 return NULL;
14408                         }
14409                         /* We come up by default in VEPA mode if SRIOV is not
14410                          * already enabled, in which case we can't force VEPA
14411                          * mode.
14412                          */
14413                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
14414                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
14415                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
14416                         }
14417                         i40e_config_bridge_mode(veb);
14418                 }
14419                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
14420                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
14421                                 veb = pf->veb[i];
14422                 }
14423                 if (!veb) {
14424                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14425                         return NULL;
14426                 }
14427
14428                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14429                 uplink_seid = veb->seid;
14430         }
14431
14432         /* get vsi sw struct */
14433         v_idx = i40e_vsi_mem_alloc(pf, type);
14434         if (v_idx < 0)
14435                 goto err_alloc;
14436         vsi = pf->vsi[v_idx];
14437         if (!vsi)
14438                 goto err_alloc;
14439         vsi->type = type;
14440         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14441
14442         if (type == I40E_VSI_MAIN)
14443                 pf->lan_vsi = v_idx;
14444         else if (type == I40E_VSI_SRIOV)
14445                 vsi->vf_id = param1;
14446         /* assign it some queues */
14447         alloc_queue_pairs = vsi->alloc_queue_pairs *
14448                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14449
14450         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14451         if (ret < 0) {
14452                 dev_info(&pf->pdev->dev,
14453                          "failed to get tracking for %d queues for VSI %d err=%d\n",
14454                          alloc_queue_pairs, vsi->seid, ret);
14455                 goto err_vsi;
14456         }
14457         vsi->base_queue = ret;
14458
14459         /* get a VSI from the hardware */
14460         vsi->uplink_seid = uplink_seid;
14461         ret = i40e_add_vsi(vsi);
14462         if (ret)
14463                 goto err_vsi;
14464
14465         switch (vsi->type) {
14466         /* setup the netdev if needed */
14467         case I40E_VSI_MAIN:
14468         case I40E_VSI_VMDQ2:
14469                 ret = i40e_config_netdev(vsi);
14470                 if (ret)
14471                         goto err_netdev;
14472                 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14473                 if (ret)
14474                         goto err_netdev;
14475                 ret = register_netdev(vsi->netdev);
14476                 if (ret)
14477                         goto err_netdev;
14478                 vsi->netdev_registered = true;
14479                 netif_carrier_off(vsi->netdev);
14480 #ifdef CONFIG_I40E_DCB
14481                 /* Setup DCB netlink interface */
14482                 i40e_dcbnl_setup(vsi);
14483 #endif /* CONFIG_I40E_DCB */
14484                 fallthrough;
14485         case I40E_VSI_FDIR:
14486                 /* set up vectors and rings if needed */
14487                 ret = i40e_vsi_setup_vectors(vsi);
14488                 if (ret)
14489                         goto err_msix;
14490
14491                 ret = i40e_alloc_rings(vsi);
14492                 if (ret)
14493                         goto err_rings;
14494
14495                 /* map all of the rings to the q_vectors */
14496                 i40e_vsi_map_rings_to_vectors(vsi);
14497
14498                 i40e_vsi_reset_stats(vsi);
14499                 break;
14500         default:
14501                 /* no netdev or rings for the other VSI types */
14502                 break;
14503         }
14504
14505         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
14506             (vsi->type == I40E_VSI_VMDQ2)) {
14507                 ret = i40e_vsi_config_rss(vsi);
14508         }
14509         return vsi;
14510
14511 err_rings:
14512         i40e_vsi_free_q_vectors(vsi);
14513 err_msix:
14514         if (vsi->netdev_registered) {
14515                 vsi->netdev_registered = false;
14516                 unregister_netdev(vsi->netdev);
14517                 free_netdev(vsi->netdev);
14518                 vsi->netdev = NULL;
14519         }
14520 err_netdev:
14521         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14522 err_vsi:
14523         i40e_vsi_clear(vsi);
14524 err_alloc:
14525         return NULL;
14526 }
14527
14528 /**
14529  * i40e_veb_get_bw_info - Query VEB BW information
14530  * @veb: the veb to query
14531  *
14532  * Query the Tx scheduler BW configuration data for given VEB
14533  **/
14534 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14535 {
14536         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14537         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14538         struct i40e_pf *pf = veb->pf;
14539         struct i40e_hw *hw = &pf->hw;
14540         u32 tc_bw_max;
14541         int ret = 0;
14542         int i;
14543
14544         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14545                                                   &bw_data, NULL);
14546         if (ret) {
14547                 dev_info(&pf->pdev->dev,
14548                          "query veb bw config failed, err %d aq_err %s\n",
14549                          ret,
14550                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14551                 goto out;
14552         }
14553
14554         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14555                                                    &ets_data, NULL);
14556         if (ret) {
14557                 dev_info(&pf->pdev->dev,
14558                          "query veb bw ets config failed, err %d aq_err %s\n",
14559                          ret,
14560                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14561                 goto out;
14562         }
14563
14564         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14565         veb->bw_max_quanta = ets_data.tc_bw_max;
14566         veb->is_abs_credits = bw_data.absolute_credits_enable;
14567         veb->enabled_tc = ets_data.tc_valid_bits;
14568         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14569                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14570         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14571                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14572                 veb->bw_tc_limit_credits[i] =
14573                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
14574                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14575         }
14576
14577 out:
14578         return ret;
14579 }
14580
14581 /**
14582  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14583  * @pf: board private structure
14584  *
14585  * On error: returns error code (negative)
14586  * On success: returns vsi index in PF (positive)
14587  **/
14588 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14589 {
14590         int ret = -ENOENT;
14591         struct i40e_veb *veb;
14592         int i;
14593
14594         /* Need to protect the allocation of switch elements at the PF level */
14595         mutex_lock(&pf->switch_mutex);
14596
14597         /* VEB list may be fragmented if VEB creation/destruction has
14598          * been happening.  We can afford to do a quick scan to look
14599          * for any free slots in the list.
14600          *
14601          * find next empty veb slot, looping back around if necessary
14602          */
14603         i = 0;
14604         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14605                 i++;
14606         if (i >= I40E_MAX_VEB) {
14607                 ret = -ENOMEM;
14608                 goto err_alloc_veb;  /* out of VEB slots! */
14609         }
14610
14611         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14612         if (!veb) {
14613                 ret = -ENOMEM;
14614                 goto err_alloc_veb;
14615         }
14616         veb->pf = pf;
14617         veb->idx = i;
14618         veb->enabled_tc = 1;
14619
14620         pf->veb[i] = veb;
14621         ret = i;
14622 err_alloc_veb:
14623         mutex_unlock(&pf->switch_mutex);
14624         return ret;
14625 }
14626
14627 /**
14628  * i40e_switch_branch_release - Delete a branch of the switch tree
14629  * @branch: where to start deleting
14630  *
14631  * This uses recursion to find the tips of the branch to be
14632  * removed, deleting until we get back to and can delete this VEB.
14633  **/
14634 static void i40e_switch_branch_release(struct i40e_veb *branch)
14635 {
14636         struct i40e_pf *pf = branch->pf;
14637         u16 branch_seid = branch->seid;
14638         u16 veb_idx = branch->idx;
14639         int i;
14640
14641         /* release any VEBs on this VEB - RECURSION */
14642         for (i = 0; i < I40E_MAX_VEB; i++) {
14643                 if (!pf->veb[i])
14644                         continue;
14645                 if (pf->veb[i]->uplink_seid == branch->seid)
14646                         i40e_switch_branch_release(pf->veb[i]);
14647         }
14648
14649         /* Release the VSIs on this VEB, but not the owner VSI.
14650          *
14651          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14652          *       the VEB itself, so don't use (*branch) after this loop.
14653          */
14654         for (i = 0; i < pf->num_alloc_vsi; i++) {
14655                 if (!pf->vsi[i])
14656                         continue;
14657                 if (pf->vsi[i]->uplink_seid == branch_seid &&
14658                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14659                         i40e_vsi_release(pf->vsi[i]);
14660                 }
14661         }
14662
14663         /* There's one corner case where the VEB might not have been
14664          * removed, so double check it here and remove it if needed.
14665          * This case happens if the veb was created from the debugfs
14666          * commands and no VSIs were added to it.
14667          */
14668         if (pf->veb[veb_idx])
14669                 i40e_veb_release(pf->veb[veb_idx]);
14670 }
14671
14672 /**
14673  * i40e_veb_clear - remove veb struct
14674  * @veb: the veb to remove
14675  **/
14676 static void i40e_veb_clear(struct i40e_veb *veb)
14677 {
14678         if (!veb)
14679                 return;
14680
14681         if (veb->pf) {
14682                 struct i40e_pf *pf = veb->pf;
14683
14684                 mutex_lock(&pf->switch_mutex);
14685                 if (pf->veb[veb->idx] == veb)
14686                         pf->veb[veb->idx] = NULL;
14687                 mutex_unlock(&pf->switch_mutex);
14688         }
14689
14690         kfree(veb);
14691 }
14692
14693 /**
14694  * i40e_veb_release - Delete a VEB and free its resources
14695  * @veb: the VEB being removed
14696  **/
14697 void i40e_veb_release(struct i40e_veb *veb)
14698 {
14699         struct i40e_vsi *vsi = NULL;
14700         struct i40e_pf *pf;
14701         int i, n = 0;
14702
14703         pf = veb->pf;
14704
14705         /* find the remaining VSI and check for extras */
14706         for (i = 0; i < pf->num_alloc_vsi; i++) {
14707                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14708                         n++;
14709                         vsi = pf->vsi[i];
14710                 }
14711         }
14712         if (n != 1) {
14713                 dev_info(&pf->pdev->dev,
14714                          "can't remove VEB %d with %d VSIs left\n",
14715                          veb->seid, n);
14716                 return;
14717         }
14718
14719         /* move the remaining VSI to uplink veb */
14720         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14721         if (veb->uplink_seid) {
14722                 vsi->uplink_seid = veb->uplink_seid;
14723                 if (veb->uplink_seid == pf->mac_seid)
14724                         vsi->veb_idx = I40E_NO_VEB;
14725                 else
14726                         vsi->veb_idx = veb->veb_idx;
14727         } else {
14728                 /* floating VEB */
14729                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14730                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14731         }
14732
14733         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14734         i40e_veb_clear(veb);
14735 }
14736
14737 /**
14738  * i40e_add_veb - create the VEB in the switch
14739  * @veb: the VEB to be instantiated
14740  * @vsi: the controlling VSI
14741  **/
14742 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14743 {
14744         struct i40e_pf *pf = veb->pf;
14745         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14746         int ret;
14747
14748         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14749                               veb->enabled_tc, false,
14750                               &veb->seid, enable_stats, NULL);
14751
14752         /* get a VEB from the hardware */
14753         if (ret) {
14754                 dev_info(&pf->pdev->dev,
14755                          "couldn't add VEB, err %d aq_err %s\n",
14756                          ret,
14757                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14758                 return -EPERM;
14759         }
14760
14761         /* get statistics counter */
14762         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14763                                          &veb->stats_idx, NULL, NULL, NULL);
14764         if (ret) {
14765                 dev_info(&pf->pdev->dev,
14766                          "couldn't get VEB statistics idx, err %d aq_err %s\n",
14767                          ret,
14768                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14769                 return -EPERM;
14770         }
14771         ret = i40e_veb_get_bw_info(veb);
14772         if (ret) {
14773                 dev_info(&pf->pdev->dev,
14774                          "couldn't get VEB bw info, err %d aq_err %s\n",
14775                          ret,
14776                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14777                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14778                 return -ENOENT;
14779         }
14780
14781         vsi->uplink_seid = veb->seid;
14782         vsi->veb_idx = veb->idx;
14783         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14784
14785         return 0;
14786 }
14787
14788 /**
14789  * i40e_veb_setup - Set up a VEB
14790  * @pf: board private structure
14791  * @flags: VEB setup flags
14792  * @uplink_seid: the switch element to link to
14793  * @vsi_seid: the initial VSI seid
14794  * @enabled_tc: Enabled TC bit-map
14795  *
14796  * This allocates the sw VEB structure and links it into the switch
14797  * It is possible and legal for this to be a duplicate of an already
14798  * existing VEB.  It is also possible for both uplink and vsi seids
14799  * to be zero, in order to create a floating VEB.
14800  *
14801  * Returns pointer to the successfully allocated VEB sw struct on
14802  * success, otherwise returns NULL on failure.
14803  **/
14804 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14805                                 u16 uplink_seid, u16 vsi_seid,
14806                                 u8 enabled_tc)
14807 {
14808         struct i40e_veb *veb, *uplink_veb = NULL;
14809         int vsi_idx, veb_idx;
14810         int ret;
14811
14812         /* if one seid is 0, the other must be 0 to create a floating relay */
14813         if ((uplink_seid == 0 || vsi_seid == 0) &&
14814             (uplink_seid + vsi_seid != 0)) {
14815                 dev_info(&pf->pdev->dev,
14816                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14817                          uplink_seid, vsi_seid);
14818                 return NULL;
14819         }
14820
14821         /* make sure there is such a vsi and uplink */
14822         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14823                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14824                         break;
14825         if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14826                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14827                          vsi_seid);
14828                 return NULL;
14829         }
14830
14831         if (uplink_seid && uplink_seid != pf->mac_seid) {
14832                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14833                         if (pf->veb[veb_idx] &&
14834                             pf->veb[veb_idx]->seid == uplink_seid) {
14835                                 uplink_veb = pf->veb[veb_idx];
14836                                 break;
14837                         }
14838                 }
14839                 if (!uplink_veb) {
14840                         dev_info(&pf->pdev->dev,
14841                                  "uplink seid %d not found\n", uplink_seid);
14842                         return NULL;
14843                 }
14844         }
14845
14846         /* get veb sw struct */
14847         veb_idx = i40e_veb_mem_alloc(pf);
14848         if (veb_idx < 0)
14849                 goto err_alloc;
14850         veb = pf->veb[veb_idx];
14851         veb->flags = flags;
14852         veb->uplink_seid = uplink_seid;
14853         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14854         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14855
14856         /* create the VEB in the switch */
14857         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14858         if (ret)
14859                 goto err_veb;
14860         if (vsi_idx == pf->lan_vsi)
14861                 pf->lan_veb = veb->idx;
14862
14863         return veb;
14864
14865 err_veb:
14866         i40e_veb_clear(veb);
14867 err_alloc:
14868         return NULL;
14869 }
14870
14871 /**
14872  * i40e_setup_pf_switch_element - set PF vars based on switch type
14873  * @pf: board private structure
14874  * @ele: element we are building info from
14875  * @num_reported: total number of elements
14876  * @printconfig: should we print the contents
14877  *
14878  * helper function to assist in extracting a few useful SEID values.
14879  **/
14880 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14881                                 struct i40e_aqc_switch_config_element_resp *ele,
14882                                 u16 num_reported, bool printconfig)
14883 {
14884         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14885         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14886         u8 element_type = ele->element_type;
14887         u16 seid = le16_to_cpu(ele->seid);
14888
14889         if (printconfig)
14890                 dev_info(&pf->pdev->dev,
14891                          "type=%d seid=%d uplink=%d downlink=%d\n",
14892                          element_type, seid, uplink_seid, downlink_seid);
14893
14894         switch (element_type) {
14895         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14896                 pf->mac_seid = seid;
14897                 break;
14898         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14899                 /* Main VEB? */
14900                 if (uplink_seid != pf->mac_seid)
14901                         break;
14902                 if (pf->lan_veb >= I40E_MAX_VEB) {
14903                         int v;
14904
14905                         /* find existing or else empty VEB */
14906                         for (v = 0; v < I40E_MAX_VEB; v++) {
14907                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14908                                         pf->lan_veb = v;
14909                                         break;
14910                                 }
14911                         }
14912                         if (pf->lan_veb >= I40E_MAX_VEB) {
14913                                 v = i40e_veb_mem_alloc(pf);
14914                                 if (v < 0)
14915                                         break;
14916                                 pf->lan_veb = v;
14917                         }
14918                 }
14919                 if (pf->lan_veb >= I40E_MAX_VEB)
14920                         break;
14921
14922                 pf->veb[pf->lan_veb]->seid = seid;
14923                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14924                 pf->veb[pf->lan_veb]->pf = pf;
14925                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14926                 break;
14927         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14928                 if (num_reported != 1)
14929                         break;
14930                 /* This is immediately after a reset so we can assume this is
14931                  * the PF's VSI
14932                  */
14933                 pf->mac_seid = uplink_seid;
14934                 pf->pf_seid = downlink_seid;
14935                 pf->main_vsi_seid = seid;
14936                 if (printconfig)
14937                         dev_info(&pf->pdev->dev,
14938                                  "pf_seid=%d main_vsi_seid=%d\n",
14939                                  pf->pf_seid, pf->main_vsi_seid);
14940                 break;
14941         case I40E_SWITCH_ELEMENT_TYPE_PF:
14942         case I40E_SWITCH_ELEMENT_TYPE_VF:
14943         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14944         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14945         case I40E_SWITCH_ELEMENT_TYPE_PE:
14946         case I40E_SWITCH_ELEMENT_TYPE_PA:
14947                 /* ignore these for now */
14948                 break;
14949         default:
14950                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14951                          element_type, seid);
14952                 break;
14953         }
14954 }
14955
14956 /**
14957  * i40e_fetch_switch_configuration - Get switch config from firmware
14958  * @pf: board private structure
14959  * @printconfig: should we print the contents
14960  *
14961  * Get the current switch configuration from the device and
14962  * extract a few useful SEID values.
14963  **/
14964 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14965 {
14966         struct i40e_aqc_get_switch_config_resp *sw_config;
14967         u16 next_seid = 0;
14968         int ret = 0;
14969         u8 *aq_buf;
14970         int i;
14971
14972         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14973         if (!aq_buf)
14974                 return -ENOMEM;
14975
14976         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14977         do {
14978                 u16 num_reported, num_total;
14979
14980                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14981                                                 I40E_AQ_LARGE_BUF,
14982                                                 &next_seid, NULL);
14983                 if (ret) {
14984                         dev_info(&pf->pdev->dev,
14985                                  "get switch config failed err %d aq_err %s\n",
14986                                  ret,
14987                                  i40e_aq_str(&pf->hw,
14988                                              pf->hw.aq.asq_last_status));
14989                         kfree(aq_buf);
14990                         return -ENOENT;
14991                 }
14992
14993                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14994                 num_total = le16_to_cpu(sw_config->header.num_total);
14995
14996                 if (printconfig)
14997                         dev_info(&pf->pdev->dev,
14998                                  "header: %d reported %d total\n",
14999                                  num_reported, num_total);
15000
15001                 for (i = 0; i < num_reported; i++) {
15002                         struct i40e_aqc_switch_config_element_resp *ele =
15003                                 &sw_config->element[i];
15004
15005                         i40e_setup_pf_switch_element(pf, ele, num_reported,
15006                                                      printconfig);
15007                 }
15008         } while (next_seid != 0);
15009
15010         kfree(aq_buf);
15011         return ret;
15012 }
15013
15014 /**
15015  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15016  * @pf: board private structure
15017  * @reinit: if the Main VSI needs to re-initialized.
15018  * @lock_acquired: indicates whether or not the lock has been acquired
15019  *
15020  * Returns 0 on success, negative value on failure
15021  **/
15022 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15023 {
15024         u16 flags = 0;
15025         int ret;
15026
15027         /* find out what's out there already */
15028         ret = i40e_fetch_switch_configuration(pf, false);
15029         if (ret) {
15030                 dev_info(&pf->pdev->dev,
15031                          "couldn't fetch switch config, err %d aq_err %s\n",
15032                          ret,
15033                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15034                 return ret;
15035         }
15036         i40e_pf_reset_stats(pf);
15037
15038         /* set the switch config bit for the whole device to
15039          * support limited promisc or true promisc
15040          * when user requests promisc. The default is limited
15041          * promisc.
15042         */
15043
15044         if ((pf->hw.pf_id == 0) &&
15045             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
15046                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15047                 pf->last_sw_conf_flags = flags;
15048         }
15049
15050         if (pf->hw.pf_id == 0) {
15051                 u16 valid_flags;
15052
15053                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15054                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15055                                                 NULL);
15056                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15057                         dev_info(&pf->pdev->dev,
15058                                  "couldn't set switch config bits, err %d aq_err %s\n",
15059                                  ret,
15060                                  i40e_aq_str(&pf->hw,
15061                                              pf->hw.aq.asq_last_status));
15062                         /* not a fatal problem, just keep going */
15063                 }
15064                 pf->last_sw_conf_valid_flags = valid_flags;
15065         }
15066
15067         /* first time setup */
15068         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
15069                 struct i40e_vsi *vsi = NULL;
15070                 u16 uplink_seid;
15071
15072                 /* Set up the PF VSI associated with the PF's main VSI
15073                  * that is already in the HW switch
15074                  */
15075                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
15076                         uplink_seid = pf->veb[pf->lan_veb]->seid;
15077                 else
15078                         uplink_seid = pf->mac_seid;
15079                 if (pf->lan_vsi == I40E_NO_VSI)
15080                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
15081                 else if (reinit)
15082                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
15083                 if (!vsi) {
15084                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15085                         i40e_cloud_filter_exit(pf);
15086                         i40e_fdir_teardown(pf);
15087                         return -EAGAIN;
15088                 }
15089         } else {
15090                 /* force a reset of TC and queue layout configurations */
15091                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
15092
15093                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
15094                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
15095                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
15096         }
15097         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
15098
15099         i40e_fdir_sb_setup(pf);
15100
15101         /* Setup static PF queue filter control settings */
15102         ret = i40e_setup_pf_filter_control(pf);
15103         if (ret) {
15104                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15105                          ret);
15106                 /* Failure here should not stop continuing other steps */
15107         }
15108
15109         /* enable RSS in the HW, even for only one queue, as the stack can use
15110          * the hash
15111          */
15112         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
15113                 i40e_pf_config_rss(pf);
15114
15115         /* fill in link information and enable LSE reporting */
15116         i40e_link_event(pf);
15117
15118         /* Initialize user-specific link properties */
15119         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
15120                                   I40E_AQ_AN_COMPLETED) ? true : false);
15121
15122         i40e_ptp_init(pf);
15123
15124         if (!lock_acquired)
15125                 rtnl_lock();
15126
15127         /* repopulate tunnel port filters */
15128         udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
15129
15130         if (!lock_acquired)
15131                 rtnl_unlock();
15132
15133         return ret;
15134 }
15135
15136 /**
15137  * i40e_determine_queue_usage - Work out queue distribution
15138  * @pf: board private structure
15139  **/
15140 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15141 {
15142         int queues_left;
15143         int q_max;
15144
15145         pf->num_lan_qps = 0;
15146
15147         /* Find the max queues to be put into basic use.  We'll always be
15148          * using TC0, whether or not DCB is running, and TC0 will get the
15149          * big RSS set.
15150          */
15151         queues_left = pf->hw.func_caps.num_tx_qp;
15152
15153         if ((queues_left == 1) ||
15154             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
15155                 /* one qp for PF, no queues for anything else */
15156                 queues_left = 0;
15157                 pf->alloc_rss_size = pf->num_lan_qps = 1;
15158
15159                 /* make sure all the fancies are disabled */
15160                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
15161                                I40E_FLAG_IWARP_ENABLED  |
15162                                I40E_FLAG_FD_SB_ENABLED  |
15163                                I40E_FLAG_FD_ATR_ENABLED |
15164                                I40E_FLAG_DCB_CAPABLE    |
15165                                I40E_FLAG_DCB_ENABLED    |
15166                                I40E_FLAG_SRIOV_ENABLED  |
15167                                I40E_FLAG_VMDQ_ENABLED);
15168                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15169         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
15170                                   I40E_FLAG_FD_SB_ENABLED |
15171                                   I40E_FLAG_FD_ATR_ENABLED |
15172                                   I40E_FLAG_DCB_CAPABLE))) {
15173                 /* one qp for PF */
15174                 pf->alloc_rss_size = pf->num_lan_qps = 1;
15175                 queues_left -= pf->num_lan_qps;
15176
15177                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
15178                                I40E_FLAG_IWARP_ENABLED  |
15179                                I40E_FLAG_FD_SB_ENABLED  |
15180                                I40E_FLAG_FD_ATR_ENABLED |
15181                                I40E_FLAG_DCB_ENABLED    |
15182                                I40E_FLAG_VMDQ_ENABLED);
15183                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15184         } else {
15185                 /* Not enough queues for all TCs */
15186                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
15187                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
15188                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
15189                                         I40E_FLAG_DCB_ENABLED);
15190                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15191                 }
15192
15193                 /* limit lan qps to the smaller of qps, cpus or msix */
15194                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15195                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15196                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15197                 pf->num_lan_qps = q_max;
15198
15199                 queues_left -= pf->num_lan_qps;
15200         }
15201
15202         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
15203                 if (queues_left > 1) {
15204                         queues_left -= 1; /* save 1 queue for FD */
15205                 } else {
15206                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
15207                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15208                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15209                 }
15210         }
15211
15212         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15213             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15214                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15215                                         (queues_left / pf->num_vf_qps));
15216                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15217         }
15218
15219         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
15220             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15221                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15222                                           (queues_left / pf->num_vmdq_qps));
15223                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15224         }
15225
15226         pf->queues_left = queues_left;
15227         dev_dbg(&pf->pdev->dev,
15228                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15229                 pf->hw.func_caps.num_tx_qp,
15230                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
15231                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15232                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15233                 queues_left);
15234 }
15235
15236 /**
15237  * i40e_setup_pf_filter_control - Setup PF static filter control
15238  * @pf: PF to be setup
15239  *
15240  * i40e_setup_pf_filter_control sets up a PF's initial filter control
15241  * settings. If PE/FCoE are enabled then it will also set the per PF
15242  * based filter sizes required for them. It also enables Flow director,
15243  * ethertype and macvlan type filter settings for the pf.
15244  *
15245  * Returns 0 on success, negative on failure
15246  **/
15247 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15248 {
15249         struct i40e_filter_control_settings *settings = &pf->filter_settings;
15250
15251         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15252
15253         /* Flow Director is enabled */
15254         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
15255                 settings->enable_fdir = true;
15256
15257         /* Ethtype and MACVLAN filters enabled for PF */
15258         settings->enable_ethtype = true;
15259         settings->enable_macvlan = true;
15260
15261         if (i40e_set_filter_control(&pf->hw, settings))
15262                 return -ENOENT;
15263
15264         return 0;
15265 }
15266
15267 #define INFO_STRING_LEN 255
15268 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
15269 static void i40e_print_features(struct i40e_pf *pf)
15270 {
15271         struct i40e_hw *hw = &pf->hw;
15272         char *buf;
15273         int i;
15274
15275         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15276         if (!buf)
15277                 return;
15278
15279         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15280 #ifdef CONFIG_PCI_IOV
15281         i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15282 #endif
15283         i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15284                       pf->hw.func_caps.num_vsis,
15285                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
15286         if (pf->flags & I40E_FLAG_RSS_ENABLED)
15287                 i += scnprintf(&buf[i], REMAIN(i), " RSS");
15288         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
15289                 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15290         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
15291                 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15292                 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15293         }
15294         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
15295                 i += scnprintf(&buf[i], REMAIN(i), " DCB");
15296         i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15297         i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15298         if (pf->flags & I40E_FLAG_PTP)
15299                 i += scnprintf(&buf[i], REMAIN(i), " PTP");
15300         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
15301                 i += scnprintf(&buf[i], REMAIN(i), " VEB");
15302         else
15303                 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15304
15305         dev_info(&pf->pdev->dev, "%s\n", buf);
15306         kfree(buf);
15307         WARN_ON(i > INFO_STRING_LEN);
15308 }
15309
15310 /**
15311  * i40e_get_platform_mac_addr - get platform-specific MAC address
15312  * @pdev: PCI device information struct
15313  * @pf: board private structure
15314  *
15315  * Look up the MAC address for the device. First we'll try
15316  * eth_platform_get_mac_address, which will check Open Firmware, or arch
15317  * specific fallback. Otherwise, we'll default to the stored value in
15318  * firmware.
15319  **/
15320 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15321 {
15322         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15323                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15324 }
15325
15326 /**
15327  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15328  * @fec_cfg: FEC option to set in flags
15329  * @flags: ptr to flags in which we set FEC option
15330  **/
15331 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
15332 {
15333         if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
15334                 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
15335         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15336             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15337                 *flags |= I40E_FLAG_RS_FEC;
15338                 *flags &= ~I40E_FLAG_BASE_R_FEC;
15339         }
15340         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15341             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15342                 *flags |= I40E_FLAG_BASE_R_FEC;
15343                 *flags &= ~I40E_FLAG_RS_FEC;
15344         }
15345         if (fec_cfg == 0)
15346                 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
15347 }
15348
15349 /**
15350  * i40e_check_recovery_mode - check if we are running transition firmware
15351  * @pf: board private structure
15352  *
15353  * Check registers indicating the firmware runs in recovery mode. Sets the
15354  * appropriate driver state.
15355  *
15356  * Returns true if the recovery mode was detected, false otherwise
15357  **/
15358 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15359 {
15360         u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15361
15362         if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15363                 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15364                 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15365                 set_bit(__I40E_RECOVERY_MODE, pf->state);
15366
15367                 return true;
15368         }
15369         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15370                 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15371
15372         return false;
15373 }
15374
15375 /**
15376  * i40e_pf_loop_reset - perform reset in a loop.
15377  * @pf: board private structure
15378  *
15379  * This function is useful when a NIC is about to enter recovery mode.
15380  * When a NIC's internal data structures are corrupted the NIC's
15381  * firmware is going to enter recovery mode.
15382  * Right after a POR it takes about 7 minutes for firmware to enter
15383  * recovery mode. Until that time a NIC is in some kind of intermediate
15384  * state. After that time period the NIC almost surely enters
15385  * recovery mode. The only way for a driver to detect intermediate
15386  * state is to issue a series of pf-resets and check a return value.
15387  * If a PF reset returns success then the firmware could be in recovery
15388  * mode so the caller of this code needs to check for recovery mode
15389  * if this function returns success. There is a little chance that
15390  * firmware will hang in intermediate state forever.
15391  * Since waiting 7 minutes is quite a lot of time this function waits
15392  * 10 seconds and then gives up by returning an error.
15393  *
15394  * Return 0 on success, negative on failure.
15395  **/
15396 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15397 {
15398         /* wait max 10 seconds for PF reset to succeed */
15399         const unsigned long time_end = jiffies + 10 * HZ;
15400         struct i40e_hw *hw = &pf->hw;
15401         int ret;
15402
15403         ret = i40e_pf_reset(hw);
15404         while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
15405                 usleep_range(10000, 20000);
15406                 ret = i40e_pf_reset(hw);
15407         }
15408
15409         if (ret == I40E_SUCCESS)
15410                 pf->pfr_count++;
15411         else
15412                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15413
15414         return ret;
15415 }
15416
15417 /**
15418  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15419  * @pf: board private structure
15420  *
15421  * Check FW registers to determine if FW issued unexpected EMP Reset.
15422  * Every time when unexpected EMP Reset occurs the FW increments
15423  * a counter of unexpected EMP Resets. When the counter reaches 10
15424  * the FW should enter the Recovery mode
15425  *
15426  * Returns true if FW issued unexpected EMP Reset
15427  **/
15428 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15429 {
15430         const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15431                            I40E_GL_FWSTS_FWS1B_MASK;
15432         return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15433                (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15434 }
15435
15436 /**
15437  * i40e_handle_resets - handle EMP resets and PF resets
15438  * @pf: board private structure
15439  *
15440  * Handle both EMP resets and PF resets and conclude whether there are
15441  * any issues regarding these resets. If there are any issues then
15442  * generate log entry.
15443  *
15444  * Return 0 if NIC is healthy or negative value when there are issues
15445  * with resets
15446  **/
15447 static int i40e_handle_resets(struct i40e_pf *pf)
15448 {
15449         const int pfr = i40e_pf_loop_reset(pf);
15450         const bool is_empr = i40e_check_fw_empr(pf);
15451
15452         if (is_empr || pfr != I40E_SUCCESS)
15453                 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15454
15455         return is_empr ? I40E_ERR_RESET_FAILED : pfr;
15456 }
15457
15458 /**
15459  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15460  * @pf: board private structure
15461  * @hw: ptr to the hardware info
15462  *
15463  * This function does a minimal setup of all subsystems needed for running
15464  * recovery mode.
15465  *
15466  * Returns 0 on success, negative on failure
15467  **/
15468 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15469 {
15470         struct i40e_vsi *vsi;
15471         int err;
15472         int v_idx;
15473
15474         pci_set_drvdata(pf->pdev, pf);
15475         pci_save_state(pf->pdev);
15476
15477         /* set up periodic task facility */
15478         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15479         pf->service_timer_period = HZ;
15480
15481         INIT_WORK(&pf->service_task, i40e_service_task);
15482         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15483
15484         err = i40e_init_interrupt_scheme(pf);
15485         if (err)
15486                 goto err_switch_setup;
15487
15488         /* The number of VSIs reported by the FW is the minimum guaranteed
15489          * to us; HW supports far more and we share the remaining pool with
15490          * the other PFs. We allocate space for more than the guarantee with
15491          * the understanding that we might not get them all later.
15492          */
15493         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15494                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15495         else
15496                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15497
15498         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15499         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15500                           GFP_KERNEL);
15501         if (!pf->vsi) {
15502                 err = -ENOMEM;
15503                 goto err_switch_setup;
15504         }
15505
15506         /* We allocate one VSI which is needed as absolute minimum
15507          * in order to register the netdev
15508          */
15509         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15510         if (v_idx < 0) {
15511                 err = v_idx;
15512                 goto err_switch_setup;
15513         }
15514         pf->lan_vsi = v_idx;
15515         vsi = pf->vsi[v_idx];
15516         if (!vsi) {
15517                 err = -EFAULT;
15518                 goto err_switch_setup;
15519         }
15520         vsi->alloc_queue_pairs = 1;
15521         err = i40e_config_netdev(vsi);
15522         if (err)
15523                 goto err_switch_setup;
15524         err = register_netdev(vsi->netdev);
15525         if (err)
15526                 goto err_switch_setup;
15527         vsi->netdev_registered = true;
15528         i40e_dbg_pf_init(pf);
15529
15530         err = i40e_setup_misc_vector_for_recovery_mode(pf);
15531         if (err)
15532                 goto err_switch_setup;
15533
15534         /* tell the firmware that we're starting */
15535         i40e_send_version(pf);
15536
15537         /* since everything's happy, start the service_task timer */
15538         mod_timer(&pf->service_timer,
15539                   round_jiffies(jiffies + pf->service_timer_period));
15540
15541         return 0;
15542
15543 err_switch_setup:
15544         i40e_reset_interrupt_capability(pf);
15545         del_timer_sync(&pf->service_timer);
15546         i40e_shutdown_adminq(hw);
15547         iounmap(hw->hw_addr);
15548         pci_disable_pcie_error_reporting(pf->pdev);
15549         pci_release_mem_regions(pf->pdev);
15550         pci_disable_device(pf->pdev);
15551         kfree(pf);
15552
15553         return err;
15554 }
15555
15556 /**
15557  * i40e_set_subsystem_device_id - set subsystem device id
15558  * @hw: pointer to the hardware info
15559  *
15560  * Set PCI subsystem device id either from a pci_dev structure or
15561  * a specific FW register.
15562  **/
15563 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15564 {
15565         struct pci_dev *pdev = ((struct i40e_pf *)hw->back)->pdev;
15566
15567         hw->subsystem_device_id = pdev->subsystem_device ?
15568                 pdev->subsystem_device :
15569                 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15570 }
15571
15572 /**
15573  * i40e_probe - Device initialization routine
15574  * @pdev: PCI device information struct
15575  * @ent: entry in i40e_pci_tbl
15576  *
15577  * i40e_probe initializes a PF identified by a pci_dev structure.
15578  * The OS initialization, configuring of the PF private structure,
15579  * and a hardware reset occur.
15580  *
15581  * Returns 0 on success, negative on failure
15582  **/
15583 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15584 {
15585         struct i40e_aq_get_phy_abilities_resp abilities;
15586 #ifdef CONFIG_I40E_DCB
15587         enum i40e_get_fw_lldp_status_resp lldp_status;
15588 #endif /* CONFIG_I40E_DCB */
15589         struct i40e_pf *pf;
15590         struct i40e_hw *hw;
15591         static u16 pfs_found;
15592         u16 wol_nvm_bits;
15593         u16 link_status;
15594 #ifdef CONFIG_I40E_DCB
15595         int status;
15596 #endif /* CONFIG_I40E_DCB */
15597         int err;
15598         u32 val;
15599         u32 i;
15600
15601         err = pci_enable_device_mem(pdev);
15602         if (err)
15603                 return err;
15604
15605         /* set up for high or low dma */
15606         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15607         if (err) {
15608                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
15609                 if (err) {
15610                         dev_err(&pdev->dev,
15611                                 "DMA configuration failed: 0x%x\n", err);
15612                         goto err_dma;
15613                 }
15614         }
15615
15616         /* set up pci connections */
15617         err = pci_request_mem_regions(pdev, i40e_driver_name);
15618         if (err) {
15619                 dev_info(&pdev->dev,
15620                          "pci_request_selected_regions failed %d\n", err);
15621                 goto err_pci_reg;
15622         }
15623
15624         pci_enable_pcie_error_reporting(pdev);
15625         pci_set_master(pdev);
15626
15627         /* Now that we have a PCI connection, we need to do the
15628          * low level device setup.  This is primarily setting up
15629          * the Admin Queue structures and then querying for the
15630          * device's current profile information.
15631          */
15632         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15633         if (!pf) {
15634                 err = -ENOMEM;
15635                 goto err_pf_alloc;
15636         }
15637         pf->next_vsi = 0;
15638         pf->pdev = pdev;
15639         set_bit(__I40E_DOWN, pf->state);
15640
15641         hw = &pf->hw;
15642         hw->back = pf;
15643
15644         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15645                                 I40E_MAX_CSR_SPACE);
15646         /* We believe that the highest register to read is
15647          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15648          * is not less than that before mapping to prevent a
15649          * kernel panic.
15650          */
15651         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15652                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15653                         pf->ioremap_len);
15654                 err = -ENOMEM;
15655                 goto err_ioremap;
15656         }
15657         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15658         if (!hw->hw_addr) {
15659                 err = -EIO;
15660                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15661                          (unsigned int)pci_resource_start(pdev, 0),
15662                          pf->ioremap_len, err);
15663                 goto err_ioremap;
15664         }
15665         hw->vendor_id = pdev->vendor;
15666         hw->device_id = pdev->device;
15667         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15668         hw->subsystem_vendor_id = pdev->subsystem_vendor;
15669         i40e_set_subsystem_device_id(hw);
15670         hw->bus.device = PCI_SLOT(pdev->devfn);
15671         hw->bus.func = PCI_FUNC(pdev->devfn);
15672         hw->bus.bus_id = pdev->bus->number;
15673         pf->instance = pfs_found;
15674
15675         /* Select something other than the 802.1ad ethertype for the
15676          * switch to use internally and drop on ingress.
15677          */
15678         hw->switch_tag = 0xffff;
15679         hw->first_tag = ETH_P_8021AD;
15680         hw->second_tag = ETH_P_8021Q;
15681
15682         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15683         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15684         INIT_LIST_HEAD(&pf->ddp_old_prof);
15685
15686         /* set up the locks for the AQ, do this only once in probe
15687          * and destroy them only once in remove
15688          */
15689         mutex_init(&hw->aq.asq_mutex);
15690         mutex_init(&hw->aq.arq_mutex);
15691
15692         pf->msg_enable = netif_msg_init(debug,
15693                                         NETIF_MSG_DRV |
15694                                         NETIF_MSG_PROBE |
15695                                         NETIF_MSG_LINK);
15696         if (debug < -1)
15697                 pf->hw.debug_mask = debug;
15698
15699         /* do a special CORER for clearing PXE mode once at init */
15700         if (hw->revision_id == 0 &&
15701             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15702                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15703                 i40e_flush(hw);
15704                 msleep(200);
15705                 pf->corer_count++;
15706
15707                 i40e_clear_pxe_mode(hw);
15708         }
15709
15710         /* Reset here to make sure all is clean and to define PF 'n' */
15711         i40e_clear_hw(hw);
15712
15713         err = i40e_set_mac_type(hw);
15714         if (err) {
15715                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15716                          err);
15717                 goto err_pf_reset;
15718         }
15719
15720         err = i40e_handle_resets(pf);
15721         if (err)
15722                 goto err_pf_reset;
15723
15724         i40e_check_recovery_mode(pf);
15725
15726         if (is_kdump_kernel()) {
15727                 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15728                 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15729         } else {
15730                 hw->aq.num_arq_entries = I40E_AQ_LEN;
15731                 hw->aq.num_asq_entries = I40E_AQ_LEN;
15732         }
15733         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15734         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15735         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15736
15737         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15738                  "%s-%s:misc",
15739                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15740
15741         err = i40e_init_shared_code(hw);
15742         if (err) {
15743                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15744                          err);
15745                 goto err_pf_reset;
15746         }
15747
15748         /* set up a default setting for link flow control */
15749         pf->hw.fc.requested_mode = I40E_FC_NONE;
15750
15751         err = i40e_init_adminq(hw);
15752         if (err) {
15753                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
15754                         dev_info(&pdev->dev,
15755                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15756                                  hw->aq.api_maj_ver,
15757                                  hw->aq.api_min_ver,
15758                                  I40E_FW_API_VERSION_MAJOR,
15759                                  I40E_FW_MINOR_VERSION(hw));
15760                 else
15761                         dev_info(&pdev->dev,
15762                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15763
15764                 goto err_pf_reset;
15765         }
15766         i40e_get_oem_version(hw);
15767
15768         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15769         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15770                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15771                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
15772                  i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15773                  hw->subsystem_vendor_id, hw->subsystem_device_id);
15774
15775         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15776             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15777                 dev_dbg(&pdev->dev,
15778                         "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15779                          hw->aq.api_maj_ver,
15780                          hw->aq.api_min_ver,
15781                          I40E_FW_API_VERSION_MAJOR,
15782                          I40E_FW_MINOR_VERSION(hw));
15783         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15784                 dev_info(&pdev->dev,
15785                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15786                          hw->aq.api_maj_ver,
15787                          hw->aq.api_min_ver,
15788                          I40E_FW_API_VERSION_MAJOR,
15789                          I40E_FW_MINOR_VERSION(hw));
15790
15791         i40e_verify_eeprom(pf);
15792
15793         /* Rev 0 hardware was never productized */
15794         if (hw->revision_id < 1)
15795                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15796
15797         i40e_clear_pxe_mode(hw);
15798
15799         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15800         if (err)
15801                 goto err_adminq_setup;
15802
15803         err = i40e_sw_init(pf);
15804         if (err) {
15805                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15806                 goto err_sw_init;
15807         }
15808
15809         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15810                 return i40e_init_recovery_mode(pf, hw);
15811
15812         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15813                                 hw->func_caps.num_rx_qp, 0, 0);
15814         if (err) {
15815                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15816                 goto err_init_lan_hmc;
15817         }
15818
15819         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15820         if (err) {
15821                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15822                 err = -ENOENT;
15823                 goto err_configure_lan_hmc;
15824         }
15825
15826         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15827          * Ignore error return codes because if it was already disabled via
15828          * hardware settings this will fail
15829          */
15830         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15831                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15832                 i40e_aq_stop_lldp(hw, true, false, NULL);
15833         }
15834
15835         /* allow a platform config to override the HW addr */
15836         i40e_get_platform_mac_addr(pdev, pf);
15837
15838         if (!is_valid_ether_addr(hw->mac.addr)) {
15839                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15840                 err = -EIO;
15841                 goto err_mac_addr;
15842         }
15843         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15844         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15845         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15846         if (is_valid_ether_addr(hw->mac.port_addr))
15847                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15848
15849         i40e_ptp_alloc_pins(pf);
15850         pci_set_drvdata(pdev, pf);
15851         pci_save_state(pdev);
15852
15853 #ifdef CONFIG_I40E_DCB
15854         status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15855         (!status &&
15856          lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15857                 (pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
15858                 (pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
15859         dev_info(&pdev->dev,
15860                  (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15861                         "FW LLDP is disabled\n" :
15862                         "FW LLDP is enabled\n");
15863
15864         /* Enable FW to write default DCB config on link-up */
15865         i40e_aq_set_dcb_parameters(hw, true, NULL);
15866
15867         err = i40e_init_pf_dcb(pf);
15868         if (err) {
15869                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15870                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15871                 /* Continue without DCB enabled */
15872         }
15873 #endif /* CONFIG_I40E_DCB */
15874
15875         /* set up periodic task facility */
15876         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15877         pf->service_timer_period = HZ;
15878
15879         INIT_WORK(&pf->service_task, i40e_service_task);
15880         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15881
15882         /* NVM bit on means WoL disabled for the port */
15883         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15884         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15885                 pf->wol_en = false;
15886         else
15887                 pf->wol_en = true;
15888         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15889
15890         /* set up the main switch operations */
15891         i40e_determine_queue_usage(pf);
15892         err = i40e_init_interrupt_scheme(pf);
15893         if (err)
15894                 goto err_switch_setup;
15895
15896         /* Reduce Tx and Rx pairs for kdump
15897          * When MSI-X is enabled, it's not allowed to use more TC queue
15898          * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15899          * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15900          */
15901         if (is_kdump_kernel())
15902                 pf->num_lan_msix = 1;
15903
15904         pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15905         pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15906         pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15907         pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15908         pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15909         pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15910                                                     UDP_TUNNEL_TYPE_GENEVE;
15911
15912         /* The number of VSIs reported by the FW is the minimum guaranteed
15913          * to us; HW supports far more and we share the remaining pool with
15914          * the other PFs. We allocate space for more than the guarantee with
15915          * the understanding that we might not get them all later.
15916          */
15917         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15918                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15919         else
15920                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15921         if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15922                 dev_warn(&pf->pdev->dev,
15923                          "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15924                          pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15925                 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15926         }
15927
15928         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15929         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15930                           GFP_KERNEL);
15931         if (!pf->vsi) {
15932                 err = -ENOMEM;
15933                 goto err_switch_setup;
15934         }
15935
15936 #ifdef CONFIG_PCI_IOV
15937         /* prep for VF support */
15938         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15939             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15940             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15941                 if (pci_num_vf(pdev))
15942                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15943         }
15944 #endif
15945         err = i40e_setup_pf_switch(pf, false, false);
15946         if (err) {
15947                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15948                 goto err_vsis;
15949         }
15950         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15951
15952         /* if FDIR VSI was set up, start it now */
15953         for (i = 0; i < pf->num_alloc_vsi; i++) {
15954                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15955                         i40e_vsi_open(pf->vsi[i]);
15956                         break;
15957                 }
15958         }
15959
15960         /* The driver only wants link up/down and module qualification
15961          * reports from firmware.  Note the negative logic.
15962          */
15963         err = i40e_aq_set_phy_int_mask(&pf->hw,
15964                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15965                                          I40E_AQ_EVENT_MEDIA_NA |
15966                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15967         if (err)
15968                 dev_info(&pf->pdev->dev, "set phy mask fail, err %d aq_err %s\n",
15969                          err,
15970                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15971
15972         /* Reconfigure hardware for allowing smaller MSS in the case
15973          * of TSO, so that we avoid the MDD being fired and causing
15974          * a reset in the case of small MSS+TSO.
15975          */
15976         val = rd32(hw, I40E_REG_MSS);
15977         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15978                 val &= ~I40E_REG_MSS_MIN_MASK;
15979                 val |= I40E_64BYTE_MSS;
15980                 wr32(hw, I40E_REG_MSS, val);
15981         }
15982
15983         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15984                 msleep(75);
15985                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15986                 if (err)
15987                         dev_info(&pf->pdev->dev, "link restart failed, err %d aq_err %s\n",
15988                                  err,
15989                                  i40e_aq_str(&pf->hw,
15990                                              pf->hw.aq.asq_last_status));
15991         }
15992         /* The main driver is (mostly) up and happy. We need to set this state
15993          * before setting up the misc vector or we get a race and the vector
15994          * ends up disabled forever.
15995          */
15996         clear_bit(__I40E_DOWN, pf->state);
15997
15998         /* In case of MSIX we are going to setup the misc vector right here
15999          * to handle admin queue events etc. In case of legacy and MSI
16000          * the misc functionality and queue processing is combined in
16001          * the same vector and that gets setup at open.
16002          */
16003         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
16004                 err = i40e_setup_misc_vector(pf);
16005                 if (err) {
16006                         dev_info(&pdev->dev,
16007                                  "setup of misc vector failed: %d\n", err);
16008                         i40e_cloud_filter_exit(pf);
16009                         i40e_fdir_teardown(pf);
16010                         goto err_vsis;
16011                 }
16012         }
16013
16014 #ifdef CONFIG_PCI_IOV
16015         /* prep for VF support */
16016         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
16017             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
16018             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
16019                 /* disable link interrupts for VFs */
16020                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
16021                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
16022                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
16023                 i40e_flush(hw);
16024
16025                 if (pci_num_vf(pdev)) {
16026                         dev_info(&pdev->dev,
16027                                  "Active VFs found, allocating resources.\n");
16028                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
16029                         if (err)
16030                                 dev_info(&pdev->dev,
16031                                          "Error %d allocating resources for existing VFs\n",
16032                                          err);
16033                 }
16034         }
16035 #endif /* CONFIG_PCI_IOV */
16036
16037         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16038                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16039                                                       pf->num_iwarp_msix,
16040                                                       I40E_IWARP_IRQ_PILE_ID);
16041                 if (pf->iwarp_base_vector < 0) {
16042                         dev_info(&pdev->dev,
16043                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
16044                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
16045                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
16046                 }
16047         }
16048
16049         i40e_dbg_pf_init(pf);
16050
16051         /* tell the firmware that we're starting */
16052         i40e_send_version(pf);
16053
16054         /* since everything's happy, start the service_task timer */
16055         mod_timer(&pf->service_timer,
16056                   round_jiffies(jiffies + pf->service_timer_period));
16057
16058         /* add this PF to client device list and launch a client service task */
16059         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16060                 err = i40e_lan_add_device(pf);
16061                 if (err)
16062                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16063                                  err);
16064         }
16065
16066 #define PCI_SPEED_SIZE 8
16067 #define PCI_WIDTH_SIZE 8
16068         /* Devices on the IOSF bus do not have this information
16069          * and will report PCI Gen 1 x 1 by default so don't bother
16070          * checking them.
16071          */
16072         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
16073                 char speed[PCI_SPEED_SIZE] = "Unknown";
16074                 char width[PCI_WIDTH_SIZE] = "Unknown";
16075
16076                 /* Get the negotiated link width and speed from PCI config
16077                  * space
16078                  */
16079                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16080                                           &link_status);
16081
16082                 i40e_set_pci_config_data(hw, link_status);
16083
16084                 switch (hw->bus.speed) {
16085                 case i40e_bus_speed_8000:
16086                         strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
16087                 case i40e_bus_speed_5000:
16088                         strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
16089                 case i40e_bus_speed_2500:
16090                         strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
16091                 default:
16092                         break;
16093                 }
16094                 switch (hw->bus.width) {
16095                 case i40e_bus_width_pcie_x8:
16096                         strlcpy(width, "8", PCI_WIDTH_SIZE); break;
16097                 case i40e_bus_width_pcie_x4:
16098                         strlcpy(width, "4", PCI_WIDTH_SIZE); break;
16099                 case i40e_bus_width_pcie_x2:
16100                         strlcpy(width, "2", PCI_WIDTH_SIZE); break;
16101                 case i40e_bus_width_pcie_x1:
16102                         strlcpy(width, "1", PCI_WIDTH_SIZE); break;
16103                 default:
16104                         break;
16105                 }
16106
16107                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16108                          speed, width);
16109
16110                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16111                     hw->bus.speed < i40e_bus_speed_8000) {
16112                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16113                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16114                 }
16115         }
16116
16117         /* get the requested speeds from the fw */
16118         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16119         if (err)
16120                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %d last_status =  %s\n",
16121                         err,
16122                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16123         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16124
16125         /* set the FEC config due to the board capabilities */
16126         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
16127
16128         /* get the supported phy types from the fw */
16129         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16130         if (err)
16131                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %d last_status =  %s\n",
16132                         err,
16133                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16134
16135         /* make sure the MFS hasn't been set lower than the default */
16136 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16137         val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
16138                I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
16139         if (val < MAX_FRAME_SIZE_DEFAULT)
16140                 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
16141                          pf->hw.port, val);
16142
16143         /* Add a filter to drop all Flow control frames from any VSI from being
16144          * transmitted. By doing so we stop a malicious VF from sending out
16145          * PAUSE or PFC frames and potentially controlling traffic for other
16146          * PF/VF VSIs.
16147          * The FW can still send Flow control frames if enabled.
16148          */
16149         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16150                                                        pf->main_vsi_seid);
16151
16152         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16153                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16154                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
16155         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16156                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
16157         /* print a string summarizing features */
16158         i40e_print_features(pf);
16159
16160         return 0;
16161
16162         /* Unwind what we've done if something failed in the setup */
16163 err_vsis:
16164         set_bit(__I40E_DOWN, pf->state);
16165         i40e_clear_interrupt_scheme(pf);
16166         kfree(pf->vsi);
16167 err_switch_setup:
16168         i40e_reset_interrupt_capability(pf);
16169         del_timer_sync(&pf->service_timer);
16170 err_mac_addr:
16171 err_configure_lan_hmc:
16172         (void)i40e_shutdown_lan_hmc(hw);
16173 err_init_lan_hmc:
16174         kfree(pf->qp_pile);
16175 err_sw_init:
16176 err_adminq_setup:
16177 err_pf_reset:
16178         iounmap(hw->hw_addr);
16179 err_ioremap:
16180         kfree(pf);
16181 err_pf_alloc:
16182         pci_disable_pcie_error_reporting(pdev);
16183         pci_release_mem_regions(pdev);
16184 err_pci_reg:
16185 err_dma:
16186         pci_disable_device(pdev);
16187         return err;
16188 }
16189
16190 /**
16191  * i40e_remove - Device removal routine
16192  * @pdev: PCI device information struct
16193  *
16194  * i40e_remove is called by the PCI subsystem to alert the driver
16195  * that is should release a PCI device.  This could be caused by a
16196  * Hot-Plug event, or because the driver is going to be removed from
16197  * memory.
16198  **/
16199 static void i40e_remove(struct pci_dev *pdev)
16200 {
16201         struct i40e_pf *pf = pci_get_drvdata(pdev);
16202         struct i40e_hw *hw = &pf->hw;
16203         int ret_code;
16204         int i;
16205
16206         i40e_dbg_pf_exit(pf);
16207
16208         i40e_ptp_stop(pf);
16209
16210         /* Disable RSS in hw */
16211         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16212         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16213
16214         /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16215          * flags, once they are set, i40e_rebuild should not be called as
16216          * i40e_prep_for_reset always returns early.
16217          */
16218         while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16219                 usleep_range(1000, 2000);
16220         set_bit(__I40E_IN_REMOVE, pf->state);
16221
16222         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
16223                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16224                 i40e_free_vfs(pf);
16225                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
16226         }
16227         /* no more scheduling of any task */
16228         set_bit(__I40E_SUSPENDED, pf->state);
16229         set_bit(__I40E_DOWN, pf->state);
16230         if (pf->service_timer.function)
16231                 del_timer_sync(&pf->service_timer);
16232         if (pf->service_task.func)
16233                 cancel_work_sync(&pf->service_task);
16234
16235         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16236                 struct i40e_vsi *vsi = pf->vsi[0];
16237
16238                 /* We know that we have allocated only one vsi for this PF,
16239                  * it was just for registering netdevice, so the interface
16240                  * could be visible in the 'ifconfig' output
16241                  */
16242                 unregister_netdev(vsi->netdev);
16243                 free_netdev(vsi->netdev);
16244
16245                 goto unmap;
16246         }
16247
16248         /* Client close must be called explicitly here because the timer
16249          * has been stopped.
16250          */
16251         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16252
16253         i40e_fdir_teardown(pf);
16254
16255         /* If there is a switch structure or any orphans, remove them.
16256          * This will leave only the PF's VSI remaining.
16257          */
16258         for (i = 0; i < I40E_MAX_VEB; i++) {
16259                 if (!pf->veb[i])
16260                         continue;
16261
16262                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
16263                     pf->veb[i]->uplink_seid == 0)
16264                         i40e_switch_branch_release(pf->veb[i]);
16265         }
16266
16267         /* Now we can shutdown the PF's VSIs, just before we kill
16268          * adminq and hmc.
16269          */
16270         for (i = pf->num_alloc_vsi; i--;)
16271                 if (pf->vsi[i]) {
16272                         i40e_vsi_close(pf->vsi[i]);
16273                         i40e_vsi_release(pf->vsi[i]);
16274                         pf->vsi[i] = NULL;
16275                 }
16276
16277         i40e_cloud_filter_exit(pf);
16278
16279         /* remove attached clients */
16280         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16281                 ret_code = i40e_lan_del_device(pf);
16282                 if (ret_code)
16283                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16284                                  ret_code);
16285         }
16286
16287         /* shutdown and destroy the HMC */
16288         if (hw->hmc.hmc_obj) {
16289                 ret_code = i40e_shutdown_lan_hmc(hw);
16290                 if (ret_code)
16291                         dev_warn(&pdev->dev,
16292                                  "Failed to destroy the HMC resources: %d\n",
16293                                  ret_code);
16294         }
16295
16296 unmap:
16297         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16298         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16299             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16300                 free_irq(pf->pdev->irq, pf);
16301
16302         /* shutdown the adminq */
16303         i40e_shutdown_adminq(hw);
16304
16305         /* destroy the locks only once, here */
16306         mutex_destroy(&hw->aq.arq_mutex);
16307         mutex_destroy(&hw->aq.asq_mutex);
16308
16309         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16310         rtnl_lock();
16311         i40e_clear_interrupt_scheme(pf);
16312         for (i = 0; i < pf->num_alloc_vsi; i++) {
16313                 if (pf->vsi[i]) {
16314                         if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16315                                 i40e_vsi_clear_rings(pf->vsi[i]);
16316                         i40e_vsi_clear(pf->vsi[i]);
16317                         pf->vsi[i] = NULL;
16318                 }
16319         }
16320         rtnl_unlock();
16321
16322         for (i = 0; i < I40E_MAX_VEB; i++) {
16323                 kfree(pf->veb[i]);
16324                 pf->veb[i] = NULL;
16325         }
16326
16327         kfree(pf->qp_pile);
16328         kfree(pf->vsi);
16329
16330         iounmap(hw->hw_addr);
16331         kfree(pf);
16332         pci_release_mem_regions(pdev);
16333
16334         pci_disable_pcie_error_reporting(pdev);
16335         pci_disable_device(pdev);
16336 }
16337
16338 /**
16339  * i40e_pci_error_detected - warning that something funky happened in PCI land
16340  * @pdev: PCI device information struct
16341  * @error: the type of PCI error
16342  *
16343  * Called to warn that something happened and the error handling steps
16344  * are in progress.  Allows the driver to quiesce things, be ready for
16345  * remediation.
16346  **/
16347 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16348                                                 pci_channel_state_t error)
16349 {
16350         struct i40e_pf *pf = pci_get_drvdata(pdev);
16351
16352         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16353
16354         if (!pf) {
16355                 dev_info(&pdev->dev,
16356                          "Cannot recover - error happened during device probe\n");
16357                 return PCI_ERS_RESULT_DISCONNECT;
16358         }
16359
16360         /* shutdown all operations */
16361         if (!test_bit(__I40E_SUSPENDED, pf->state))
16362                 i40e_prep_for_reset(pf);
16363
16364         /* Request a slot reset */
16365         return PCI_ERS_RESULT_NEED_RESET;
16366 }
16367
16368 /**
16369  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16370  * @pdev: PCI device information struct
16371  *
16372  * Called to find if the driver can work with the device now that
16373  * the pci slot has been reset.  If a basic connection seems good
16374  * (registers are readable and have sane content) then return a
16375  * happy little PCI_ERS_RESULT_xxx.
16376  **/
16377 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16378 {
16379         struct i40e_pf *pf = pci_get_drvdata(pdev);
16380         pci_ers_result_t result;
16381         u32 reg;
16382
16383         dev_dbg(&pdev->dev, "%s\n", __func__);
16384         if (pci_enable_device_mem(pdev)) {
16385                 dev_info(&pdev->dev,
16386                          "Cannot re-enable PCI device after reset.\n");
16387                 result = PCI_ERS_RESULT_DISCONNECT;
16388         } else {
16389                 pci_set_master(pdev);
16390                 pci_restore_state(pdev);
16391                 pci_save_state(pdev);
16392                 pci_wake_from_d3(pdev, false);
16393
16394                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16395                 if (reg == 0)
16396                         result = PCI_ERS_RESULT_RECOVERED;
16397                 else
16398                         result = PCI_ERS_RESULT_DISCONNECT;
16399         }
16400
16401         return result;
16402 }
16403
16404 /**
16405  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16406  * @pdev: PCI device information struct
16407  */
16408 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16409 {
16410         struct i40e_pf *pf = pci_get_drvdata(pdev);
16411
16412         i40e_prep_for_reset(pf);
16413 }
16414
16415 /**
16416  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16417  * @pdev: PCI device information struct
16418  */
16419 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16420 {
16421         struct i40e_pf *pf = pci_get_drvdata(pdev);
16422
16423         if (test_bit(__I40E_IN_REMOVE, pf->state))
16424                 return;
16425
16426         i40e_reset_and_rebuild(pf, false, false);
16427 #ifdef CONFIG_PCI_IOV
16428         i40e_restore_all_vfs_msi_state(pdev);
16429 #endif /* CONFIG_PCI_IOV */
16430 }
16431
16432 /**
16433  * i40e_pci_error_resume - restart operations after PCI error recovery
16434  * @pdev: PCI device information struct
16435  *
16436  * Called to allow the driver to bring things back up after PCI error
16437  * and/or reset recovery has finished.
16438  **/
16439 static void i40e_pci_error_resume(struct pci_dev *pdev)
16440 {
16441         struct i40e_pf *pf = pci_get_drvdata(pdev);
16442
16443         dev_dbg(&pdev->dev, "%s\n", __func__);
16444         if (test_bit(__I40E_SUSPENDED, pf->state))
16445                 return;
16446
16447         i40e_handle_reset_warning(pf, false);
16448 }
16449
16450 /**
16451  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16452  * using the mac_address_write admin q function
16453  * @pf: pointer to i40e_pf struct
16454  **/
16455 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16456 {
16457         struct i40e_hw *hw = &pf->hw;
16458         u8 mac_addr[6];
16459         u16 flags = 0;
16460         int ret;
16461
16462         /* Get current MAC address in case it's an LAA */
16463         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16464                 ether_addr_copy(mac_addr,
16465                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16466         } else {
16467                 dev_err(&pf->pdev->dev,
16468                         "Failed to retrieve MAC address; using default\n");
16469                 ether_addr_copy(mac_addr, hw->mac.addr);
16470         }
16471
16472         /* The FW expects the mac address write cmd to first be called with
16473          * one of these flags before calling it again with the multicast
16474          * enable flags.
16475          */
16476         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16477
16478         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16479                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16480
16481         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16482         if (ret) {
16483                 dev_err(&pf->pdev->dev,
16484                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16485                 return;
16486         }
16487
16488         flags = I40E_AQC_MC_MAG_EN
16489                         | I40E_AQC_WOL_PRESERVE_ON_PFR
16490                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16491         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16492         if (ret)
16493                 dev_err(&pf->pdev->dev,
16494                         "Failed to enable Multicast Magic Packet wake up\n");
16495 }
16496
16497 /**
16498  * i40e_shutdown - PCI callback for shutting down
16499  * @pdev: PCI device information struct
16500  **/
16501 static void i40e_shutdown(struct pci_dev *pdev)
16502 {
16503         struct i40e_pf *pf = pci_get_drvdata(pdev);
16504         struct i40e_hw *hw = &pf->hw;
16505
16506         set_bit(__I40E_SUSPENDED, pf->state);
16507         set_bit(__I40E_DOWN, pf->state);
16508
16509         del_timer_sync(&pf->service_timer);
16510         cancel_work_sync(&pf->service_task);
16511         i40e_cloud_filter_exit(pf);
16512         i40e_fdir_teardown(pf);
16513
16514         /* Client close must be called explicitly here because the timer
16515          * has been stopped.
16516          */
16517         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16518
16519         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16520                 i40e_enable_mc_magic_wake(pf);
16521
16522         i40e_prep_for_reset(pf);
16523
16524         wr32(hw, I40E_PFPM_APM,
16525              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16526         wr32(hw, I40E_PFPM_WUFC,
16527              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16528
16529         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16530         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16531             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16532                 free_irq(pf->pdev->irq, pf);
16533
16534         /* Since we're going to destroy queues during the
16535          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16536          * whole section
16537          */
16538         rtnl_lock();
16539         i40e_clear_interrupt_scheme(pf);
16540         rtnl_unlock();
16541
16542         if (system_state == SYSTEM_POWER_OFF) {
16543                 pci_wake_from_d3(pdev, pf->wol_en);
16544                 pci_set_power_state(pdev, PCI_D3hot);
16545         }
16546 }
16547
16548 /**
16549  * i40e_suspend - PM callback for moving to D3
16550  * @dev: generic device information structure
16551  **/
16552 static int __maybe_unused i40e_suspend(struct device *dev)
16553 {
16554         struct i40e_pf *pf = dev_get_drvdata(dev);
16555         struct i40e_hw *hw = &pf->hw;
16556
16557         /* If we're already suspended, then there is nothing to do */
16558         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16559                 return 0;
16560
16561         set_bit(__I40E_DOWN, pf->state);
16562
16563         /* Ensure service task will not be running */
16564         del_timer_sync(&pf->service_timer);
16565         cancel_work_sync(&pf->service_task);
16566
16567         /* Client close must be called explicitly here because the timer
16568          * has been stopped.
16569          */
16570         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16571
16572         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16573                 i40e_enable_mc_magic_wake(pf);
16574
16575         /* Since we're going to destroy queues during the
16576          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16577          * whole section
16578          */
16579         rtnl_lock();
16580
16581         i40e_prep_for_reset(pf);
16582
16583         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16584         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16585
16586         /* Clear the interrupt scheme and release our IRQs so that the system
16587          * can safely hibernate even when there are a large number of CPUs.
16588          * Otherwise hibernation might fail when mapping all the vectors back
16589          * to CPU0.
16590          */
16591         i40e_clear_interrupt_scheme(pf);
16592
16593         rtnl_unlock();
16594
16595         return 0;
16596 }
16597
16598 /**
16599  * i40e_resume - PM callback for waking up from D3
16600  * @dev: generic device information structure
16601  **/
16602 static int __maybe_unused i40e_resume(struct device *dev)
16603 {
16604         struct i40e_pf *pf = dev_get_drvdata(dev);
16605         int err;
16606
16607         /* If we're not suspended, then there is nothing to do */
16608         if (!test_bit(__I40E_SUSPENDED, pf->state))
16609                 return 0;
16610
16611         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16612          * since we're going to be restoring queues
16613          */
16614         rtnl_lock();
16615
16616         /* We cleared the interrupt scheme when we suspended, so we need to
16617          * restore it now to resume device functionality.
16618          */
16619         err = i40e_restore_interrupt_scheme(pf);
16620         if (err) {
16621                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16622                         err);
16623         }
16624
16625         clear_bit(__I40E_DOWN, pf->state);
16626         i40e_reset_and_rebuild(pf, false, true);
16627
16628         rtnl_unlock();
16629
16630         /* Clear suspended state last after everything is recovered */
16631         clear_bit(__I40E_SUSPENDED, pf->state);
16632
16633         /* Restart the service task */
16634         mod_timer(&pf->service_timer,
16635                   round_jiffies(jiffies + pf->service_timer_period));
16636
16637         return 0;
16638 }
16639
16640 static const struct pci_error_handlers i40e_err_handler = {
16641         .error_detected = i40e_pci_error_detected,
16642         .slot_reset = i40e_pci_error_slot_reset,
16643         .reset_prepare = i40e_pci_error_reset_prepare,
16644         .reset_done = i40e_pci_error_reset_done,
16645         .resume = i40e_pci_error_resume,
16646 };
16647
16648 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16649
16650 static struct pci_driver i40e_driver = {
16651         .name     = i40e_driver_name,
16652         .id_table = i40e_pci_tbl,
16653         .probe    = i40e_probe,
16654         .remove   = i40e_remove,
16655         .driver   = {
16656                 .pm = &i40e_pm_ops,
16657         },
16658         .shutdown = i40e_shutdown,
16659         .err_handler = &i40e_err_handler,
16660         .sriov_configure = i40e_pci_sriov_configure,
16661 };
16662
16663 /**
16664  * i40e_init_module - Driver registration routine
16665  *
16666  * i40e_init_module is the first routine called when the driver is
16667  * loaded. All it does is register with the PCI subsystem.
16668  **/
16669 static int __init i40e_init_module(void)
16670 {
16671         int err;
16672
16673         pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16674         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16675
16676         /* There is no need to throttle the number of active tasks because
16677          * each device limits its own task using a state bit for scheduling
16678          * the service task, and the device tasks do not interfere with each
16679          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16680          * since we need to be able to guarantee forward progress even under
16681          * memory pressure.
16682          */
16683         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16684         if (!i40e_wq) {
16685                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16686                 return -ENOMEM;
16687         }
16688
16689         i40e_dbg_init();
16690         err = pci_register_driver(&i40e_driver);
16691         if (err) {
16692                 destroy_workqueue(i40e_wq);
16693                 i40e_dbg_exit();
16694                 return err;
16695         }
16696
16697         return 0;
16698 }
16699 module_init(i40e_init_module);
16700
16701 /**
16702  * i40e_exit_module - Driver exit cleanup routine
16703  *
16704  * i40e_exit_module is called just before the driver is removed
16705  * from memory.
16706  **/
16707 static void __exit i40e_exit_module(void)
16708 {
16709         pci_unregister_driver(&i40e_driver);
16710         destroy_workqueue(i40e_wq);
16711         ida_destroy(&i40e_client_ida);
16712         i40e_dbg_exit();
16713 }
16714 module_exit(i40e_exit_module);