src/miniublk: add user recovery

We are going to test ublk's user recovery feature so add support in
miniublk.

Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>

diff --git a/src/miniublk.c b/src/miniublk.c
index fe1029130de34bf32db1e3aa395f80e531b6953e..1c97668e9ff3dc52ba72fd198029def5aa1e378a 100644 (file)
--- a/src/miniublk.c
+++ b/src/miniublk.c
@@ -17,6 +17,8 @@
 #include <pthread.h>
 #include <getopt.h>
 #include <limits.h>
+#include <string.h>
+#include <errno.h>
 #include <sys/syscall.h>
 #include <sys/mman.h>
 #include <sys/ioctl.h>
@@ -74,6 +76,7 @@ struct ublk_tgt_ops {
        int (*queue_io)(struct ublk_queue *, int tag);
        void (*tgt_io_done)(struct ublk_queue *,
                        int tag, const struct io_uring_cqe *);
+       int (*recover_tgt)(struct ublk_dev *);
 };
 
 struct ublk_tgt {
@@ -372,6 +375,29 @@ static int ublk_ctrl_get_params(struct ublk_dev *dev,
        return __ublk_ctrl_cmd(dev, &data);
 }
 
+static int ublk_ctrl_start_user_recover(struct ublk_dev *dev)
+{
+       struct ublk_ctrl_cmd_data data = {
+               .cmd_op = UBLK_CMD_START_USER_RECOVERY,
+               .flags  = 0,
+       };
+
+       return __ublk_ctrl_cmd(dev, &data);
+}
+
+static int ublk_ctrl_end_user_recover(struct ublk_dev *dev,
+               int daemon_pid)
+{
+       struct ublk_ctrl_cmd_data data = {
+               .cmd_op = UBLK_CMD_END_USER_RECOVERY,
+               .flags  = CTRL_CMD_HAS_DATA,
+       };
+
+       dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
+
+       return __ublk_ctrl_cmd(dev, &data);
+}
+
 static const char *ublk_dev_state_desc(struct ublk_dev *dev)
 {
        switch (dev->dev_info.state) {
@@ -379,6 +405,8 @@ static const char *ublk_dev_state_desc(struct ublk_dev *dev)
                return "DEAD";
        case UBLK_S_DEV_LIVE:
                return "LIVE";
+       case UBLK_S_DEV_QUIESCED:
+               return "QUIESCED";
        default:
                return "UNKNOWN";
        };
@@ -550,9 +578,12 @@ static int ublk_dev_prep(struct ublk_dev *dev)
                goto fail;
        }
 
-       if (dev->tgt.ops->init_tgt)
+       if (dev->dev_info.state != UBLK_S_DEV_QUIESCED && dev->tgt.ops->init_tgt)
                ret = dev->tgt.ops->init_tgt(dev);
 
+       else if (dev->dev_info.state == UBLK_S_DEV_QUIESCED && dev->tgt.ops->recover_tgt)
+               ret = dev->tgt.ops->recover_tgt(dev);
+
        return ret;
 fail:
        close(dev->fds[0]);
@@ -831,7 +862,7 @@ static void ublk_set_parameters(struct ublk_dev *dev)
                                dev->dev_info.dev_id, ret);
 }
 
-static int ublk_start_daemon(struct ublk_dev *dev)
+static int ublk_start_daemon(struct ublk_dev *dev, bool recovery)
 {
        int ret, i;
        void *thread_ret;
@@ -853,12 +884,22 @@ static int ublk_start_daemon(struct ublk_dev *dev)
                                &dev->q[i]);
        }
 
-       ublk_set_parameters(dev);
 
        /* everything is fine now, start us */
-       ret = ublk_ctrl_start_dev(dev, getpid());
-       if (ret < 0)
-               goto fail;
+       if (recovery) {
+               ret = ublk_ctrl_end_user_recover(dev, getpid());
+               if (ret < 0) {
+                       ublk_err("%s: ublk_ctrl_end_user_recover failed: %d\n", __func__, ret);
+                       goto fail;
+               }
+       } else {
+               ublk_set_parameters(dev);
+               ret = ublk_ctrl_start_dev(dev, getpid());
+               if (ret < 0) {
+                       ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret);
+                       goto fail;
+               }
+       }
 
        ublk_ctrl_get_info(dev);
        ublk_ctrl_dump(dev, true);
@@ -880,6 +921,7 @@ static int cmd_dev_add(int argc, char *argv[])
                { "number",             1,      NULL, 'n' },
                { "queues",             1,      NULL, 'q' },
                { "depth",              1,      NULL, 'd' },
+               { "recovery",           0,      NULL, 'r' },
                { "debug_mask", 1,      NULL, 0},
                { "quiet",      0,      NULL, 0},
                { NULL }
@@ -891,8 +933,9 @@ static int cmd_dev_add(int argc, char *argv[])
        const char *tgt_type = NULL;
        int dev_id = -1;
        unsigned nr_queues = 2, depth = UBLK_QUEUE_DEPTH;
+       int user_recovery = 0;
 
-       while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
+       while ((opt = getopt_long(argc, argv, "-:t:n:d:q:r",
                                  longopts, &option_idx)) != -1) {
                switch (opt) {
                case 'n':
@@ -907,6 +950,9 @@ static int cmd_dev_add(int argc, char *argv[])
                case 'd':
                        depth = strtol(optarg, NULL, 10);
                        break;
+               case 'r':
+                       user_recovery = 1;
+                       break;
                case 0:
                        if (!strcmp(longopts[option_idx].name, "debug_mask"))
                                ublk_dbg_mask = strtol(optarg, NULL, 16);
@@ -942,6 +988,8 @@ static int cmd_dev_add(int argc, char *argv[])
        info->dev_id = dev_id;
         info->nr_hw_queues = nr_queues;
         info->queue_depth = depth;
+       if (user_recovery)
+               info->flags |= UBLK_F_USER_RECOVERY;
        dev->tgt.ops = ops;
        dev->tgt.argc = argc;
        dev->tgt.argv = argv;
@@ -953,7 +1001,95 @@ static int cmd_dev_add(int argc, char *argv[])
                goto fail;
        }
 
-       ret = ublk_start_daemon(dev);
+       ret = ublk_start_daemon(dev, false);
+       if (ret < 0) {
+               ublk_err("%s: can't start daemon id %d, type %s\n",
+                               __func__, dev_id, tgt_type);
+               goto fail_del;
+       }
+
+fail_del:
+       ublk_ctrl_del_dev(dev);
+fail:
+       ublk_ctrl_deinit(dev);
+       return ret;
+}
+
+static int cmd_dev_recover(int argc, char *argv[])
+{
+       static const struct option longopts[] = {
+               { "type",               1,      NULL, 't' },
+               { "number",             1,      NULL, 'n' },
+               { "debug_mask", 1,      NULL, 0},
+               { "quiet",      0,      NULL, 0},
+               { NULL }
+       };
+       const struct ublk_tgt_ops *ops;
+       struct ublksrv_ctrl_dev_info *info;
+       struct ublk_dev *dev;
+       int ret, option_idx, opt;
+       const char *tgt_type = NULL;
+       int dev_id = -1;
+
+       while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
+                                 longopts, &option_idx)) != -1) {
+               switch (opt) {
+               case 'n':
+                       dev_id = strtol(optarg, NULL, 10);
+                       break;
+               case 't':
+                       tgt_type = optarg;
+                       break;
+               case 0:
+                       if (!strcmp(longopts[option_idx].name, "debug_mask"))
+                               ublk_dbg_mask = strtol(optarg, NULL, 16);
+                       if (!strcmp(longopts[option_idx].name, "quiet"))
+                               ublk_dbg_mask = 0;
+                       break;
+               }
+       }
+
+       optind = 0;
+
+       ops = ublk_find_tgt(tgt_type);
+       if (!ops) {
+               ublk_err("%s: no such tgt type, type %s\n",
+                               __func__, tgt_type);
+               return -ENODEV;
+       }
+
+       dev = ublk_ctrl_init();
+       if (!dev) {
+               ublk_err("%s: can't alloc dev id %d, type %s\n",
+                               __func__, dev_id, tgt_type);
+               return -ENOMEM;
+       }
+
+       info = &dev->dev_info;
+       info->dev_id = dev_id;
+       ret = ublk_ctrl_get_info(dev);
+       if (ret < 0) {
+               ublk_err("%s: can't get dev info from %d\n", __func__, dev_id);
+               goto fail;
+       }
+
+       ret = ublk_ctrl_get_params(dev, &dev->tgt.params);
+       if (ret) {
+               ublk_err("dev %d set basic parameter failed %d\n",
+                               dev->dev_info.dev_id, ret);
+               goto fail;
+       }
+
+       dev->tgt.ops = ops;
+       dev->tgt.argc = argc;
+       dev->tgt.argv = argv;
+       ret = ublk_ctrl_start_user_recover(dev);
+       if (ret < 0) {
+               ublk_err("%s: can't start recovery for %d\n", __func__, dev_id);
+               goto fail;
+       }
+
+       ret = ublk_start_daemon(dev, true);
        if (ret < 0) {
                ublk_err("%s: can't start daemon id %d, type %s\n",
                                __func__, dev_id, tgt_type);
@@ -1125,7 +1261,9 @@ static int cmd_dev_help(int argc, char *argv[])
        printf("\t -a delete all devices -n delete specified device\n");
        printf("%s list [-n dev_id] -a \n", argv[0]);
        printf("\t -a list all devices, -n list specified device, default -a \n");
-
+       printf("%s recover -t {null|loop} [-n dev_id] \n", argv[0]);
+       printf("\t -t loop -f backing_file \n");
+       printf("\t -t null\n");
        return 0;
 }
 
@@ -1150,6 +1288,12 @@ static int ublk_null_tgt_init(struct ublk_dev *dev)
        return 0;
 }
 
+static int ublk_null_tgt_recover(struct ublk_dev *dev)
+{
+       dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
+       return 0;
+}
+
 static int ublk_null_queue_io(struct ublk_queue *q, int tag)
 {
        const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
@@ -1264,15 +1408,17 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
                }
        }
 
-       ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);
-
-       if (!file)
+       if (!file) {
+               ublk_err( "%s: backing file is unset!\n", __func__);
                return -EINVAL;
+       }
+
+       ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);
 
        fd = open(file, O_RDWR);
        if (fd < 0) {
-               ublk_err( "%s: backing file %s can't be opened\n",
-                               __func__, file);
+               ublk_err("%s: backing file %s can't be opened: %s\n",
+                               __func__, file, strerror(errno));
                return -EBADF;
        }
 
@@ -1301,7 +1447,8 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
        if (fcntl(fd, F_SETFL, O_DIRECT)) {
                p.basic.logical_bs_shift = 9;
                p.basic.physical_bs_shift = 12;
-               ublk_log("%s: ublk-loop fallback to buffered IO\n", __func__);
+               ublk_log("%s: %s, ublk-loop fallback to buffered IO\n",
+                               __func__, strerror(errno));
        }
 
        dev->tgt.dev_size = bytes;
@@ -1313,11 +1460,100 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
        return 0;
 }
 
+static int ublk_loop_tgt_recover(struct ublk_dev *dev)
+{
+       static const struct option lo_longopts[] = {
+               { "file",               1,      NULL, 'f' },
+               { NULL }
+       };
+       unsigned long long bytes;
+       char **argv = dev->tgt.argv;
+       int argc = dev->tgt.argc;
+       struct ublk_params p = dev->tgt.params;
+       char *file = NULL;
+       int fd, opt;
+       unsigned int bs = 1 << 9, pbs = 1 << 12;
+       struct stat st;
+
+       while ((opt = getopt_long(argc, argv, "-:f:",
+                                 lo_longopts, NULL)) != -1) {
+               switch (opt) {
+               case 'f':
+                       file = strdup(optarg);
+                       break;
+               }
+       }
+
+       if (!file) {
+               ublk_err( "%s: backing file is unset!\n", __func__);
+               return -EINVAL;
+       }
+
+       ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);
+
+       fd = open(file, O_RDWR);
+       if (fd < 0) {
+               ublk_err( "%s: backing file %s can't be opened: %s\n",
+                               __func__, file, strerror(errno));
+               return -EBADF;
+       }
+
+       if (fstat(fd, &st) < 0) {
+               close(fd);
+               return -EBADF;
+       }
+
+       if (S_ISBLK(st.st_mode)) {
+               if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
+                       return -EBADF;
+               if (ioctl(fd, BLKSSZGET, &bs) != 0)
+                       return -1;
+               if (ioctl(fd, BLKPBSZGET, &pbs) != 0)
+                       return -1;
+       } else if (S_ISREG(st.st_mode)) {
+               bytes = st.st_size;
+       } else {
+               bytes = 0;
+       }
+
+       if (fcntl(fd, F_SETFL, O_DIRECT)) {
+               /* buffered I/O */
+               ublk_log("%s: %s, ublk-loop fallback to buffered IO\n",
+                               __func__, strerror(errno));
+       }
+       else {
+               /* direct I/O */
+               if (p.basic.logical_bs_shift != ilog2(bs)) {
+                       ublk_err("%s: logical block size should be %d, I got %d\n",
+                                       __func__, 1 << p.basic.logical_bs_shift, bs);
+                       return -1;
+               }
+               if (p.basic.physical_bs_shift != ilog2(pbs)) {
+                       ublk_err("%s: physical block size should be %d, I got %d\n",
+                                       __func__, 1 << p.basic.physical_bs_shift, pbs);
+                       return -1;
+               }
+       }
+
+       if (p.basic.dev_sectors << 9 != bytes) {
+               ublk_err("%s: device size should be %lld, I got %lld\n",
+                               __func__, p.basic.dev_sectors << 9, bytes);
+               return -1;
+       }
+
+       dev->tgt.dev_size = bytes;
+       dev->fds[1] = fd;
+       dev->nr_fds += 1;
+
+       return 0;
+}
+
 const struct ublk_tgt_ops tgt_ops_list[] = {
        {
                .name = "null",
                .init_tgt = ublk_null_tgt_init,
                .queue_io = ublk_null_queue_io,
+               .recover_tgt = ublk_null_tgt_recover,
        },
 
        {
@@ -1326,6 +1562,7 @@ const struct ublk_tgt_ops tgt_ops_list[] = {
                .deinit_tgt = ublk_loop_tgt_deinit,
                .queue_io = ublk_loop_queue_io,
                .tgt_io_done = ublk_loop_io_done,
+               .recover_tgt = ublk_loop_tgt_recover,
        },
 };
 
@@ -1359,6 +1596,8 @@ int main(int argc, char *argv[])
                ret = cmd_dev_list(argc, argv);
        else if (!strcmp(cmd, "help"))
                ret = cmd_dev_help(argc, argv);
+       else if (!strcmp(cmd, "recover"))
+               ret = cmd_dev_recover(argc, argv);
 out:
        if (ret)
                cmd_dev_help(argc, argv);