hdev = hci_pi(sk)->hdev;
        if (hdev) {
-               if (hci_pi(sk)->channel == HCI_CHANNEL_USER) {
+               if (hci_pi(sk)->channel == HCI_CHANNEL_USER &&
+                   !hci_dev_test_flag(hdev, HCI_UNREGISTER)) {
                        /* When releasing a user channel exclusive access,
                         * call hci_dev_do_close directly instead of calling
                         * hci_dev_close to ensure the exclusive access will
                         * The checking of HCI_AUTO_OFF is not needed in this
                         * case since it will have been cleared already when
                         * opening the user channel.
+                        *
+                        * Make sure to also check that we haven't already
+                        * unregistered since all the cleanup will have already
+                        * been complete and hdev will get released when we put
+                        * below.
                         */
                        hci_dev_do_close(hdev);
                        hci_dev_clear_flag(hdev, HCI_USER_CHANNEL);