struct arm_smmu_device *smmu;
        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_master *master;
+       struct arm_smmu_cd *cdptr;
 
        if (!fwspec)
                return -ENOENT;
        if (ret)
                return ret;
 
+       if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+               cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
+               if (!cdptr)
+                       return -ENOMEM;
+       }
+
        /*
         * Prevent arm_smmu_share_asid() from trying to change the ASID
         * of either the old or new domain while we are working on it.
        switch (smmu_domain->stage) {
        case ARM_SMMU_DOMAIN_S1: {
                struct arm_smmu_cd target_cd;
-               struct arm_smmu_cd *cdptr;
-
-               cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
-               if (!cdptr) {
-                       ret = -ENOMEM;
-                       goto out_list_del;
-               }
 
                arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
                arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
        }
 
        arm_smmu_enable_ats(master, smmu_domain);
-       goto out_unlock;
-
-out_list_del:
-       spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-       list_del_init(&master->domain_head);
-       spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
-
-out_unlock:
        mutex_unlock(&arm_smmu_asid_lock);
-       return ret;
+       return 0;
 }
 
 static int arm_smmu_attach_dev_ste(struct device *dev,