if (desc->affinity_notify) {
                kref_get(&desc->affinity_notify->kref);
-               schedule_work(&desc->affinity_notify->work);
+               if (!schedule_work(&desc->affinity_notify->work)) {
+                       /* Work was already scheduled, drop our extra ref */
+                       kref_put(&desc->affinity_notify->kref,
+                                desc->affinity_notify->release);
+               }
        }
        irqd_set(data, IRQD_AFFINITY_SET);
 
        raw_spin_unlock_irqrestore(&desc->lock, flags);
 
        if (old_notify) {
-               cancel_work_sync(&old_notify->work);
+               if (cancel_work_sync(&old_notify->work)) {
+                       /* Pending work had a ref, put that one too */
+                       kref_put(&old_notify->kref, old_notify->release);
+               }
                kref_put(&old_notify->kref, old_notify->release);
        }