bcachefs: New bucket invalidate path

In the old allocator code, preparing an existing empty bucket was part
of the same code path that invalidated buckets containing cached data.
In the new allocator code this is no longer the case: the main allocator
path finds empty buckets (via the new freespace btree), and can't
allocate buckets that contain cached data.

We now need a separate code path to invalidate buckets containing cached
data when we're low on empty buckets, which this patch implements. When
the number of free buckets decreases that triggers the new invalidate
path to run, which uses the LRU btree to pick cached data buckets to
invalidate until we're above our watermark.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 9514c2e5f01e3ed1138dde807d07a462781971eb..fac9337dc5434df2bfd17bc98a3ae7d09b95f228 100644 (file)
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -718,6 +718,86 @@ void bch2_do_discards(struct bch_fs *c)
                percpu_ref_put(&c->writes);
 }
 
+static int invalidate_one_bucket(struct btree_trans *trans, struct bch_dev *ca)
+{
+       struct bch_fs *c = trans->c;
+       struct btree_iter lru_iter, alloc_iter = { NULL };
+       struct bkey_s_c k;
+       struct bkey_i_alloc_v4 *a;
+       u64 bucket, idx;
+       int ret;
+
+       bch2_trans_iter_init(trans, &lru_iter, BTREE_ID_lru,
+                            POS(ca->dev_idx, 0), 0);
+       k = bch2_btree_iter_peek(&lru_iter);
+       ret = bkey_err(k);
+       if (ret)
+               goto out;
+
+       if (!k.k || k.k->p.inode != ca->dev_idx)
+               goto out;
+
+       if (bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_lru, c,
+                                   "non lru key in lru btree"))
+               goto out;
+
+       idx     = k.k->p.offset;
+       bucket  = le64_to_cpu(bkey_s_c_to_lru(k).v->idx);
+
+       a = bch2_trans_start_alloc_update(trans, &alloc_iter,
+                                         POS(ca->dev_idx, bucket));
+       ret = PTR_ERR_OR_ZERO(a);
+       if (ret)
+               goto out;
+
+       if (bch2_fs_inconsistent_on(idx != alloc_lru_idx(a->v), c,
+                       "invalidating bucket with wrong lru idx (got %llu should be %llu",
+                       idx, alloc_lru_idx(a->v)))
+               goto out;
+
+       SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false);
+       a->v.gen++;
+       a->v.data_type          = 0;
+       a->v.dirty_sectors      = 0;
+       a->v.cached_sectors     = 0;
+       a->v.io_time[READ]      = atomic64_read(&c->io_clock[READ].now);
+       a->v.io_time[WRITE]     = atomic64_read(&c->io_clock[WRITE].now);
+
+       ret = bch2_trans_update(trans, &alloc_iter, &a->k_i,
+                               BTREE_TRIGGER_BUCKET_INVALIDATE);
+out:
+       bch2_trans_iter_exit(trans, &alloc_iter);
+       bch2_trans_iter_exit(trans, &lru_iter);
+       return ret;
+}
+
+static void bch2_do_invalidates_work(struct work_struct *work)
+{
+       struct bch_fs *c = container_of(work, struct bch_fs, invalidate_work);
+       struct bch_dev *ca;
+       struct btree_trans trans;
+       unsigned i;
+       int ret = 0;
+
+       bch2_trans_init(&trans, c, 0, 0);
+
+       for_each_member_device(ca, c, i)
+               while (!ret && should_invalidate_buckets(ca))
+                       ret = __bch2_trans_do(&trans, NULL, NULL,
+                                             BTREE_INSERT_USE_RESERVE|
+                                             BTREE_INSERT_NOFAIL,
+                                       invalidate_one_bucket(&trans, ca));
+
+       bch2_trans_exit(&trans);
+       percpu_ref_put(&c->writes);
+}
+
+void bch2_do_invalidates(struct bch_fs *c)
+{
+       if (percpu_ref_tryget(&c->writes))
+               queue_work(system_long_wq, &c->invalidate_work);
+}
+
 static int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca)
 {
        struct btree_trans trans;
@@ -1002,4 +1082,5 @@ void bch2_fs_allocator_background_init(struct bch_fs *c)
 {
        spin_lock_init(&c->freelist_lock);
        INIT_WORK(&c->discard_work, bch2_do_discards_work);
+       INIT_WORK(&c->invalidate_work, bch2_do_invalidates_work);
 }

diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h
index 8ba9bf853c2fe787614d40f9eaa49451ea564e04..d4883d3cd642dc823c992e1df8446c98cba4c6cf 100644 (file)
--- a/fs/bcachefs/alloc_background.h
+++ b/fs/bcachefs/alloc_background.h
@@ -115,6 +115,17 @@ int bch2_trans_mark_alloc(struct btree_trans *, struct bkey_s_c,
                          struct bkey_i *, unsigned);
 void bch2_do_discards(struct bch_fs *);
 
+static inline bool should_invalidate_buckets(struct bch_dev *ca)
+{
+       struct bch_dev_usage u = bch2_dev_usage_read(ca);
+
+       return u.d[BCH_DATA_cached].buckets &&
+               u.buckets_unavailable + u.d[BCH_DATA_cached].buckets <
+               ca->mi.nbuckets >> 7;
+}
+
+void bch2_do_invalidates(struct bch_fs *);
+
 int bch2_fs_freespace_init(struct bch_fs *);
 
 void bch2_recalc_capacity(struct bch_fs *);

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index ca48b3f863042436dbba7a8de75de2463da1e45b..66d9c209252e7c933ad88e1d7af231b1823e2b3c 100644 (file)
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -759,6 +759,7 @@ struct bch_fs {
 
        struct buckets_waiting_for_journal buckets_waiting_for_journal;
        struct work_struct      discard_work;
+       struct work_struct      invalidate_work;
 
        /* GARBAGE COLLECTION */
        struct task_struct      *gc_thread;

diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 0e86b45b6c55d367615d51ce0d33e207e23de723..bfab5d88550bbe147227bc53bb14cff641af510a 100644 (file)
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -548,6 +548,11 @@ int bch2_mark_alloc(struct btree_trans *trans,
            !new_a.journal_seq)
                bch2_do_discards(c);
 
+       if (!old_a.data_type &&
+           new_a.data_type &&
+           should_invalidate_buckets(ca))
+               bch2_do_invalidates(c);
+
        if (bucket_state(new_a) == BUCKET_need_gc_gens) {
                atomic_inc(&c->kick_gc);
                wake_up_process(c->gc_thread);