&instance->irq_context[i])) {
                        dev_err(&instance->pdev->dev,
                                "Failed to register IRQ for vector %d.\n", i);
-                       for (j = 0; j < i; j++)
+                       for (j = 0; j < i; j++) {
+                               if (j < instance->low_latency_index_start)
+                                       irq_set_affinity_hint(
+                                               pci_irq_vector(pdev, j), NULL);
                                free_irq(pci_irq_vector(pdev, j),
                                         &instance->irq_context[j]);
+                       }
                        /* Retry irq register for IO_APIC*/
                        instance->msix_vectors = 0;
                        instance->msix_load_balance = false;
 
        if (instance->msix_vectors)
                for (i = 0; i < instance->msix_vectors; i++) {
+                       if (i < instance->low_latency_index_start)
+                               irq_set_affinity_hint(
+                                   pci_irq_vector(instance->pdev, i), NULL);
                        free_irq(pci_irq_vector(instance->pdev, i),
                                 &instance->irq_context[i]);
                }