file: add fput and file_ref_put routines optimized for use when closing a fd

Vast majority of the time closing a file descriptor also operates on the
last reference, where a regular fput usage will result in 2 atomics.
This can be changed to only suffer 1.

See commentary above file_ref_put_close() for more information.

Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
Link: https://lore.kernel.org/r/20250305123644.554845-2-mjguzik@gmail.com
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/fs/file.c b/fs/file.c
index d868cdb95d1e78668fa561b14bb8b4b3554eabfa..3fef798b96e5b67627182061e650d4b81092bbf0 100644 (file)
--- a/fs/file.c
+++ b/fs/file.c
@@ -26,6 +26,28 @@
 
 #include "internal.h"
 
+bool __file_ref_put_badval(file_ref_t *ref, unsigned long cnt)
+{
+       /*
+        * If the reference count was already in the dead zone, then this
+        * put() operation is imbalanced. Warn, put the reference count back to
+        * DEAD and tell the caller to not deconstruct the object.
+        */
+       if (WARN_ONCE(cnt >= FILE_REF_RELEASED, "imbalanced put on file reference count")) {
+               atomic_long_set(&ref->refcnt, FILE_REF_DEAD);
+               return false;
+       }
+
+       /*
+        * This is a put() operation on a saturated refcount. Restore the
+        * mean saturation value and tell the caller to not deconstruct the
+        * object.
+        */
+       if (cnt > FILE_REF_MAXREF)
+               atomic_long_set(&ref->refcnt, FILE_REF_SATURATED);
+       return false;
+}
+
 /**
  * __file_ref_put - Slowpath of file_ref_put()
  * @ref:       Pointer to the reference count
@@ -67,24 +89,7 @@ bool __file_ref_put(file_ref_t *ref, unsigned long cnt)
                return true;
        }
 
-       /*
-        * If the reference count was already in the dead zone, then this
-        * put() operation is imbalanced. Warn, put the reference count back to
-        * DEAD and tell the caller to not deconstruct the object.
-        */
-       if (WARN_ONCE(cnt >= FILE_REF_RELEASED, "imbalanced put on file reference count")) {
-               atomic_long_set(&ref->refcnt, FILE_REF_DEAD);
-               return false;
-       }
-
-       /*
-        * This is a put() operation on a saturated refcount. Restore the
-        * mean saturation value and tell the caller to not deconstruct the
-        * object.
-        */
-       if (cnt > FILE_REF_MAXREF)
-               atomic_long_set(&ref->refcnt, FILE_REF_SATURATED);
-       return false;
+       return __file_ref_put_badval(ref, cnt);
 }
 EXPORT_SYMBOL_GPL(__file_ref_put);
 

diff --git a/fs/file_table.c b/fs/file_table.c
index f0291a66f9db4184359a52da2f4ead308de4abfe..18debb7bd28502ce04f017b6ca6fc272d87f95b3 100644 (file)
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -495,31 +495,37 @@ void flush_delayed_fput(void)
 }
 EXPORT_SYMBOL_GPL(flush_delayed_fput);
 
-void fput(struct file *file)
+static void __fput_deferred(struct file *file)
 {
-       if (file_ref_put(&file->f_ref)) {
-               struct task_struct *task = current;
+       struct task_struct *task = current;
+
+       if (unlikely(!(file->f_mode & (FMODE_BACKING | FMODE_OPENED)))) {
+               file_free(file);
+               return;
+       }
 
-               if (unlikely(!(file->f_mode & (FMODE_BACKING | FMODE_OPENED)))) {
-                       file_free(file);
+       if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) {
+               init_task_work(&file->f_task_work, ____fput);
+               if (!task_work_add(task, &file->f_task_work, TWA_RESUME))
                        return;
-               }
-               if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) {
-                       init_task_work(&file->f_task_work, ____fput);
-                       if (!task_work_add(task, &file->f_task_work, TWA_RESUME))
-                               return;
-                       /*
-                        * After this task has run exit_task_work(),
-                        * task_work_add() will fail.  Fall through to delayed
-                        * fput to avoid leaking *file.
-                        */
-               }
-
-               if (llist_add(&file->f_llist, &delayed_fput_list))
-                       schedule_delayed_work(&delayed_fput_work, 1);
+               /*
+                * After this task has run exit_task_work(),
+                * task_work_add() will fail.  Fall through to delayed
+                * fput to avoid leaking *file.
+                */
        }
+
+       if (llist_add(&file->f_llist, &delayed_fput_list))
+               schedule_delayed_work(&delayed_fput_work, 1);
 }
 
+void fput(struct file *file)
+{
+       if (unlikely(file_ref_put(&file->f_ref)))
+               __fput_deferred(file);
+}
+EXPORT_SYMBOL(fput);
+
 /*
  * synchronous analog of fput(); for kernel threads that might be needed
  * in some umount() (and thus can't use flush_delayed_fput() without
@@ -533,10 +539,32 @@ void __fput_sync(struct file *file)
        if (file_ref_put(&file->f_ref))
                __fput(file);
 }
-
-EXPORT_SYMBOL(fput);
 EXPORT_SYMBOL(__fput_sync);
 
+/*
+ * Equivalent to __fput_sync(), but optimized for being called with the last
+ * reference.
+ *
+ * See file_ref_put_close() for details.
+ */
+void fput_close_sync(struct file *file)
+{
+       if (likely(file_ref_put_close(&file->f_ref)))
+               __fput(file);
+}
+
+/*
+ * Equivalent to fput(), but optimized for being called with the last
+ * reference.
+ *
+ * See file_ref_put_close() for details.
+ */
+void fput_close(struct file *file)
+{
+       if (file_ref_put_close(&file->f_ref))
+               __fput_deferred(file);
+}
+
 void __init files_init(void)
 {
        struct kmem_cache_args args = {

diff --git a/fs/internal.h b/fs/internal.h
index e7f02ae1e0981bd129a995c02f3b7a5defd7c669..05c817f39a28aafd1d94878ad73023453357a076 100644 (file)
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -118,6 +118,9 @@ static inline void put_file_access(struct file *file)
        }
 }
 
+void fput_close_sync(struct file *);
+void fput_close(struct file *);
+
 /*
  * super.c
  */

diff --git a/include/linux/file_ref.h b/include/linux/file_ref.h
index 9b3a8d9b17ab77fc3732507ce0d9940922877858..6ef92d765a6684cb2b5ec632f7106c00ba00eee3 100644 (file)
--- a/include/linux/file_ref.h
+++ b/include/linux/file_ref.h
@@ -61,6 +61,7 @@ static inline void file_ref_init(file_ref_t *ref, unsigned long cnt)
        atomic_long_set(&ref->refcnt, cnt - 1);
 }
 
+bool __file_ref_put_badval(file_ref_t *ref, unsigned long cnt);
 bool __file_ref_put(file_ref_t *ref, unsigned long cnt);
 
 /**
@@ -160,6 +161,39 @@ static __always_inline __must_check bool file_ref_put(file_ref_t *ref)
        return __file_ref_put(ref, cnt);
 }
 
+/**
+ * file_ref_put_close - drop a reference expecting it would transition to FILE_REF_NOREF
+ * @ref:       Pointer to the reference count
+ *
+ * Semantically it is equivalent to calling file_ref_put(), but it trades lower
+ * performance in face of other CPUs also modifying the refcount for higher
+ * performance when this happens to be the last reference.
+ *
+ * For the last reference file_ref_put() issues 2 atomics. One to drop the
+ * reference and another to transition it to FILE_REF_DEAD. This routine does
+ * the work in one step, but in order to do it has to pre-read the variable which
+ * decreases scalability.
+ *
+ * Use with close() et al, stick to file_ref_put() by default.
+ */
+static __always_inline __must_check bool file_ref_put_close(file_ref_t *ref)
+{
+       long old, new;
+
+       old = atomic_long_read(&ref->refcnt);
+       do {
+               if (unlikely(old < 0))
+                       return __file_ref_put_badval(ref, old);
+
+               if (old == FILE_REF_ONEREF)
+                       new = FILE_REF_DEAD;
+               else
+                       new = old - 1;
+       } while (!atomic_long_try_cmpxchg(&ref->refcnt, &old, new));
+
+       return new == FILE_REF_DEAD;
+}
+
 /**
  * file_ref_read - Read the number of file references
  * @ref: Pointer to the reference count