#define EXOFS_MIN_PID 0x10000 /* Smallest partition ID */
#define EXOFS_OBJ_OFF 0x10000 /* offset for objects */
#define EXOFS_SUPER_ID 0x10000 /* object ID for on-disk superblock */
+#define EXOFS_DEVTABLE_ID 0x10001 /* object ID for on-disk device table */
#define EXOFS_ROOT_ID 0x10002 /* object ID for root directory */
/* exofs Application specific page/attribute */
#define EXOFS_SUPER_MAGIC 0x5DF5
/*
- * The file system control block - stored in an object's data (mainly, the one
- * with ID EXOFS_SUPER_ID). This is where the in-memory superblock is stored
- * on disk. Right now it just has a magic value, which is basically a sanity
- * check on our ability to communicate with the object store.
+ * The file system control block - stored in object EXOFS_SUPER_ID's data.
+ * This is where the in-memory superblock is stored on disk.
*/
+enum {EXOFS_FSCB_VER = 1, EXOFS_DT_VER = 1};
struct exofs_fscb {
__le64 s_nextid; /* Highest object ID used */
- __le32 s_numfiles; /* Number of files on fs */
+ __le64 s_numfiles; /* Number of files on fs */
+ __le32 s_version; /* == EXOFS_FSCB_VER */
__le16 s_magic; /* Magic signature */
__le16 s_newfs; /* Non-zero if this is a new fs */
-};
+
+ /* From here on it's a static part, only written by mkexofs */
+ __le64 s_dev_table_oid; /* Resurved, not used */
+ __le64 s_dev_table_count; /* == 0 means no dev_table */
+} __packed;
+
+/*
+ * Describes the raid used in the FS. It is part of the device table.
+ * This here is taken from the pNFS-objects definition. In exofs we
+ * use one raid policy through-out the filesystem. (NOTE: the funny
+ * alignment at begining. We take care of it at exofs_device_table.
+ */
+struct exofs_dt_data_map {
+ __le32 cb_num_comps;
+ __le64 cb_stripe_unit;
+ __le32 cb_group_width;
+ __le32 cb_group_depth;
+ __le32 cb_mirror_cnt;
+ __le32 cb_raid_algorithm;
+} __packed;
+
+/*
+ * This is an osd device information descriptor. It is a single entry in
+ * the exofs device table. It describes an osd target lun which
+ * contains data belonging to this FS. (Same partition_id on all devices)
+ */
+struct exofs_dt_device_info {
+ __le32 systemid_len;
+ u8 systemid[OSD_SYSTEMID_LEN];
+ __le64 long_name_offset; /* If !0 then offset-in-file */
+ __le32 osdname_len; /* */
+ u8 osdname[44]; /* Embbeded, Ususally an asci uuid */
+} __packed;
+
+/*
+ * The EXOFS device table - stored in object EXOFS_DEVTABLE_ID's data.
+ * It contains the raid used for this multy-device FS and an array of
+ * participating devices.
+ */
+struct exofs_device_table {
+ __le32 dt_version; /* == EXOFS_DT_VER */
+ struct exofs_dt_data_map dt_data_map; /* Raid policy to use */
+
+ /* Resurved space For future use. Total includeing this:
+ * (8 * sizeof(le64))
+ */
+ __le64 __Resurved[4];
+
+ __le64 dt_num_devices; /* Array size */
+ struct exofs_dt_device_info dt_dev_table[]; /* Array of devices */
+} __packed;
/****************************************************************************
* inode-related things
/*TODO: Maybe use kmem_cach per sbi of size
* exofs_io_state_size(sbi->s_numdevs)
*/
- ios = kzalloc(exofs_io_state_size(1), GFP_KERNEL);
+ ios = kzalloc(exofs_io_state_size(sbi->s_numdevs), GFP_KERNEL);
if (unlikely(!ios)) {
*pios = NULL;
return -ENOMEM;
{
int i, ret;
- for (i = 0; i < 1; i++) {
+ for (i = 0; i < ios->sbi->s_numdevs; i++) {
struct osd_request *or;
- or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+ or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
if (unlikely(!or)) {
EXOFS_ERR("%s: osd_start_request failed\n", __func__);
ret = -ENOMEM;
{
int i, ret;
- for (i = 0; i < 1; i++) {
+ for (i = 0; i < ios->sbi->s_numdevs; i++) {
struct osd_request *or;
- or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+ or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
if (unlikely(!or)) {
EXOFS_ERR("%s: osd_start_request failed\n", __func__);
ret = -ENOMEM;
{
int i, ret;
- for (i = 0; i < 1; i++) {
+ for (i = 0; i < ios->sbi->s_numdevs; i++) {
struct osd_request *or;
- or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+ or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
if (unlikely(!or)) {
EXOFS_ERR("%s: osd_start_request failed\n", __func__);
ret = -ENOMEM;
if (ios->bio) {
struct bio *bio;
- bio = ios->bio;
+ if (i != 0) {
+ bio = bio_kmalloc(GFP_KERNEL,
+ ios->bio->bi_max_vecs);
+ if (unlikely(!bio)) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ __bio_clone(bio, ios->bio);
+ bio->bi_bdev = NULL;
+ bio->bi_next = NULL;
+ ios->per_dev[i].bio = bio;
+ } else {
+ bio = ios->bio;
+ }
osd_req_write(or, &ios->obj, ios->offset, bio,
ios->length);
for (i = 0; i < 1; i++) {
struct osd_request *or;
+ unsigned first_dev = (unsigned)ios->obj.id;
- or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+ first_dev %= ios->sbi->s_numdevs;
+ or = osd_start_request(ios->sbi->s_ods[first_dev], GFP_KERNEL);
if (unlikely(!or)) {
EXOFS_ERR("%s: osd_start_request failed\n", __func__);
ret = -ENOMEM;
attr = g_attr_logical_length;
attr.val_ptr = &newsize;
- for (i = 0; i < 1; i++) {
+ for (i = 0; i < sbi->s_numdevs; i++) {
struct osd_request *or;
- or = osd_start_request(sbi->s_dev, GFP_KERNEL);
+ or = osd_start_request(sbi->s_ods[i], GFP_KERNEL);
if (unlikely(!or)) {
EXOFS_ERR("%s: osd_start_request failed\n", __func__);
ret = -ENOMEM;
if (ret)
goto out;
- ios->length = sizeof(*fscb);
+ /* Note: We only write the changing part of the fscb. .i.e upto the
+ * the fscb->s_dev_table_oid member. There is no read-modify-write
+ * here.
+ */
+ ios->length = offsetof(struct exofs_fscb, s_dev_table_oid);
memset(fscb, 0, ios->length);
fscb->s_nextid = cpu_to_le64(sbi->s_nextid);
fscb->s_numfiles = cpu_to_le32(sbi->s_numfiles);
fscb->s_magic = cpu_to_le16(sb->s_magic);
fscb->s_newfs = 0;
+ fscb->s_version = EXOFS_FSCB_VER;
ios->obj.id = EXOFS_SUPER_ID;
ios->offset = 0;
msg, dev_path ?: "", odi->osdname, _LLU(pid));
}
+void exofs_free_sbi(struct exofs_sb_info *sbi)
+{
+ while (sbi->s_numdevs) {
+ int i = --sbi->s_numdevs;
+ struct osd_dev *od = sbi->s_ods[i];
+
+ if (od) {
+ sbi->s_ods[i] = NULL;
+ osduld_put_device(od);
+ }
+ }
+ kfree(sbi);
+}
+
/*
* This function is called when the vfs is freeing the superblock. We just
* need to free our own part.
msecs_to_jiffies(100));
}
- _exofs_print_device("Unmounting", NULL, sbi->s_dev, sbi->s_pid);
- osduld_put_device(sbi->s_dev);
- kfree(sb->s_fs_info);
+ _exofs_print_device("Unmounting", NULL, sbi->s_ods[0], sbi->s_pid);
+
+ exofs_free_sbi(sbi);
sb->s_fs_info = NULL;
}
+static int _read_and_match_data_map(struct exofs_sb_info *sbi, unsigned numdevs,
+ struct exofs_device_table *dt)
+{
+ sbi->data_map.odm_num_comps =
+ le32_to_cpu(dt->dt_data_map.cb_num_comps);
+ sbi->data_map.odm_stripe_unit =
+ le64_to_cpu(dt->dt_data_map.cb_stripe_unit);
+ sbi->data_map.odm_group_width =
+ le32_to_cpu(dt->dt_data_map.cb_group_width);
+ sbi->data_map.odm_group_depth =
+ le32_to_cpu(dt->dt_data_map.cb_group_depth);
+ sbi->data_map.odm_mirror_cnt =
+ le32_to_cpu(dt->dt_data_map.cb_mirror_cnt);
+ sbi->data_map.odm_raid_algorithm =
+ le32_to_cpu(dt->dt_data_map.cb_raid_algorithm);
+
+/* FIXME: Hard coded mirror only for now. if not so do not mount */
+ if ((sbi->data_map.odm_num_comps != numdevs) ||
+ (sbi->data_map.odm_stripe_unit != EXOFS_BLKSIZE) ||
+ (sbi->data_map.odm_raid_algorithm != PNFS_OSD_RAID_0) ||
+ (sbi->data_map.odm_mirror_cnt != (numdevs - 1)))
+ return -EINVAL;
+ else
+ return 0;
+}
+
+/* @odi is valid only as long as @fscb_dev is valid */
+static int exofs_devs_2_odi(struct exofs_dt_device_info *dt_dev,
+ struct osd_dev_info *odi)
+{
+ odi->systemid_len = le32_to_cpu(dt_dev->systemid_len);
+ memcpy(odi->systemid, dt_dev->systemid, odi->systemid_len);
+
+ odi->osdname_len = le32_to_cpu(dt_dev->osdname_len);
+ odi->osdname = dt_dev->osdname;
+
+ /* FIXME support long names. Will need a _put function */
+ if (dt_dev->long_name_offset)
+ return -EINVAL;
+
+ /* Make sure osdname is printable!
+ * mkexofs should give us space for a null-terminator else the
+ * device-table is invalid.
+ */
+ if (unlikely(odi->osdname_len >= sizeof(dt_dev->osdname)))
+ odi->osdname_len = sizeof(dt_dev->osdname) - 1;
+ dt_dev->osdname[odi->osdname_len] = 0;
+
+ /* If it's all zeros something is bad we read past end-of-obj */
+ return !(odi->systemid_len || odi->osdname_len);
+}
+
+static int exofs_read_lookup_dev_table(struct exofs_sb_info **psbi,
+ unsigned table_count)
+{
+ struct exofs_sb_info *sbi = *psbi;
+ struct osd_dev *fscb_od;
+ struct osd_obj_id obj = {.partition = sbi->s_pid,
+ .id = EXOFS_DEVTABLE_ID};
+ struct exofs_device_table *dt;
+ unsigned table_bytes = table_count * sizeof(dt->dt_dev_table[0]) +
+ sizeof(*dt);
+ unsigned numdevs, i;
+ int ret;
+
+ dt = kmalloc(table_bytes, GFP_KERNEL);
+ if (unlikely(!dt)) {
+ EXOFS_ERR("ERROR: allocating %x bytes for device table\n",
+ table_bytes);
+ return -ENOMEM;
+ }
+
+ fscb_od = sbi->s_ods[0];
+ sbi->s_ods[0] = NULL;
+ sbi->s_numdevs = 0;
+ ret = exofs_read_kern(fscb_od, sbi->s_cred, &obj, 0, dt, table_bytes);
+ if (unlikely(ret)) {
+ EXOFS_ERR("ERROR: reading device table\n");
+ goto out;
+ }
+
+ numdevs = le64_to_cpu(dt->dt_num_devices);
+ if (unlikely(!numdevs)) {
+ ret = -EINVAL;
+ goto out;
+ }
+ WARN_ON(table_count != numdevs);
+
+ ret = _read_and_match_data_map(sbi, numdevs, dt);
+ if (unlikely(ret))
+ goto out;
+
+ if (likely(numdevs > 1)) {
+ unsigned size = numdevs * sizeof(sbi->s_ods[0]);
+
+ sbi = krealloc(sbi, sizeof(*sbi) + size, GFP_KERNEL);
+ if (unlikely(!sbi)) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memset(&sbi->s_ods[1], 0, size - sizeof(sbi->s_ods[0]));
+ *psbi = sbi;
+ }
+
+ for (i = 0; i < numdevs; i++) {
+ struct exofs_fscb fscb;
+ struct osd_dev_info odi;
+ struct osd_dev *od;
+
+ if (exofs_devs_2_odi(&dt->dt_dev_table[i], &odi)) {
+ EXOFS_ERR("ERROR: Read all-zeros device entry\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ printk(KERN_NOTICE "Add device[%d]: osd_name-%s\n",
+ i, odi.osdname);
+
+ /* On all devices the device table is identical. The user can
+ * specify any one of the participating devices on the command
+ * line. We always keep them in device-table order.
+ */
+ if (fscb_od && osduld_device_same(fscb_od, &odi)) {
+ sbi->s_ods[i] = fscb_od;
+ ++sbi->s_numdevs;
+ fscb_od = NULL;
+ continue;
+ }
+
+ od = osduld_info_lookup(&odi);
+ if (unlikely(IS_ERR(od))) {
+ ret = PTR_ERR(od);
+ EXOFS_ERR("ERROR: device requested is not found "
+ "osd_name-%s =>%d\n", odi.osdname, ret);
+ goto out;
+ }
+
+ sbi->s_ods[i] = od;
+ ++sbi->s_numdevs;
+
+ /* Read the fscb of the other devices to make sure the FS
+ * partition is there.
+ */
+ ret = exofs_read_kern(od, sbi->s_cred, &obj, 0, &fscb,
+ sizeof(fscb));
+ if (unlikely(ret)) {
+ EXOFS_ERR("ERROR: Malformed participating device "
+ "error reading fscb osd_name-%s\n",
+ odi.osdname);
+ goto out;
+ }
+
+ /* TODO: verify other information is correct and FS-uuid
+ * matches. Benny what did you say about device table
+ * generation and old devices?
+ */
+ }
+
+out:
+ kfree(dt);
+ if (unlikely(!ret && fscb_od)) {
+ EXOFS_ERR(
+ "ERROR: Bad device-table container device not present\n");
+ osduld_put_device(fscb_od);
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
/*
* Read the superblock from the OSD and fill in the fields
*/
struct osd_dev *od; /* Master device */
struct exofs_fscb fscb; /*on-disk superblock info */
struct osd_obj_id obj;
+ unsigned table_count;
int ret;
sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
goto free_sbi;
}
- sbi->s_dev = od;
+ sbi->s_ods[0] = od;
+ sbi->s_numdevs = 1;
sbi->s_pid = opts->pid;
sbi->s_timeout = opts->timeout;
ret = -EINVAL;
goto free_sbi;
}
+ if (le32_to_cpu(fscb.s_version) != EXOFS_FSCB_VER) {
+ EXOFS_ERR("ERROR: Bad FSCB version expected-%d got-%d\n",
+ EXOFS_FSCB_VER, le32_to_cpu(fscb.s_version));
+ ret = -EINVAL;
+ goto free_sbi;
+ }
/* start generation numbers from a random point */
get_random_bytes(&sbi->s_next_generation, sizeof(u32));
spin_lock_init(&sbi->s_next_gen_lock);
+ table_count = le64_to_cpu(fscb.s_dev_table_count);
+ if (table_count) {
+ ret = exofs_read_lookup_dev_table(&sbi, table_count);
+ if (unlikely(ret))
+ goto free_sbi;
+ }
+
/* set up operation vectors */
sb->s_fs_info = sbi;
sb->s_op = &exofs_sops;
goto free_sbi;
}
- _exofs_print_device("Mounting", opts->dev_name, sbi->s_dev, sbi->s_pid);
+ _exofs_print_device("Mounting", opts->dev_name, sbi->s_ods[0],
+ sbi->s_pid);
return 0;
free_sbi:
EXOFS_ERR("Unable to mount exofs on %s pid=0x%llx err=%d\n",
opts->dev_name, sbi->s_pid, ret);
- osduld_put_device(sbi->s_dev); /* NULL safe */
- kfree(sbi);
+ exofs_free_sbi(sbi);
return ret;
}
projects / users / willy / pagecache.git / commitdiff