I believe this can only happen in the case of a corrupted filesystem.
So -EIO looks like the appropriate error.
Signed-off-by: J. Bruce Fields <bfields@redhat.com>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
                                dput(new);
                                return ERR_PTR(-EIO);
                        }
+                       if (d_ancestor(new, dentry)) {
+                               spin_unlock(&inode->i_lock);
+                               dput(new);
+                               return ERR_PTR(-EIO);
+                       }
                        write_seqlock(&rename_lock);
                        __d_materialise_dentry(dentry, new);
                        write_sequnlock(&rename_lock);