mm/damon/core: introduce damon_call()

Introduce a new DAMON core API function, damon_call().  It aims to replace
some damon_callback usages that access damon_ctx of ongoing kdamond with
additional synchronizations.  It receives a function pointer, let the
parallel kdamond invokes the function, and returns after the invocation is
finished, or canceled due to some races.

kdamond invokes the function inside the main loop after sampling is done.
If it is deactivated by DAMOS watermarks or already out of the main loop,
mark the request as canceled so that damon_call() can wakeup and return.

Link: https://lkml.kernel.org/r/20250103174400.54890-4-sj@kernel.org
Signed-off-by: SeongJae Park <sj@kernel.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/damon.h b/include/linux/damon.h
index a67f2c4940e94dbaf1f26a96119891cac59894bb..ac2d42a50751c76c94f8e338146718ee5722fbe3 100644 (file)
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -552,6 +552,27 @@ struct damon_callback {
        void (*before_terminate)(struct damon_ctx *context);
 };
 
+/*
+ * struct damon_call_control - Control damon_call().
+ *
+ * @fn:                        Function to be called back.
+ * @data:              Data that will be passed to @fn.
+ * @return_code:       Return code from @fn invocation.
+ *
+ * Control damon_call(), which requests specific kdamond to invoke a given
+ * function.  Refer to damon_call() for more details.
+ */
+struct damon_call_control {
+       int (*fn)(void *data);
+       void *data;
+       int return_code;
+/* private: internal use only */
+       /* informs if the kdamond finished handling of the request */
+       struct completion completion;
+       /* informs if the kdamond canceled @fn infocation */
+       bool canceled;
+};
+
 /**
  * struct damon_attrs - Monitoring attributes for accuracy/overhead control.
  *
@@ -632,6 +653,9 @@ struct damon_ctx {
        /* for scheme quotas prioritization */
        unsigned long *regions_score_histogram;
 
+       struct damon_call_control *call_control;
+       struct mutex call_control_lock;
+
 /* public: */
        struct task_struct *kdamond;
        struct mutex kdamond_lock;
@@ -779,6 +803,8 @@ static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs
 int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive);
 int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
 
+int damon_call(struct damon_ctx *ctx, struct damon_call_control *control);
+
 int damon_set_region_biggest_system_ram_default(struct damon_target *t,
                                unsigned long *start, unsigned long *end);
 

diff --git a/mm/damon/core.c b/mm/damon/core.c
index 5192ee29f6cfd657449a44c38895caeb8b0a97c6..97f19ec4179c78eb7d021776924738660f81949a 100644 (file)
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -504,6 +504,7 @@ struct damon_ctx *damon_new_ctx(void)
        ctx->next_ops_update_sis = 0;
 
        mutex_init(&ctx->kdamond_lock);
+       mutex_init(&ctx->call_control_lock);
 
        ctx->attrs.min_nr_regions = 10;
        ctx->attrs.max_nr_regions = 1000;
@@ -1162,6 +1163,54 @@ int damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
        return err;
 }
 
+static bool damon_is_running(struct damon_ctx *ctx)
+{
+       bool running;
+
+       mutex_lock(&ctx->kdamond_lock);
+       running = ctx->kdamond != NULL;
+       mutex_unlock(&ctx->kdamond_lock);
+       return running;
+}
+
+/**
+ * damon_call() - Invoke a given function on DAMON worker thread (kdamond).
+ * @ctx:       DAMON context to call the function for.
+ * @control:   Control variable of the call request.
+ *
+ * Ask DAMON worker thread (kdamond) of @ctx to call a function with an
+ * argument data that respectively passed via &damon_call_control->fn and
+ * &damon_call_control->data of @control, and wait until the kdamond finishes
+ * handling of the request.
+ *
+ * The kdamond executes the function with the argument in the main loop, just
+ * after a sampling of the iteration is finished.  The function can hence
+ * safely access the internal data of the &struct damon_ctx without additional
+ * synchronization.  The return value of the function will be saved in
+ * &damon_call_control->return_code.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
+{
+       init_completion(&control->completion);
+       control->canceled = false;
+
+       mutex_lock(&ctx->call_control_lock);
+       if (ctx->call_control) {
+               mutex_unlock(&ctx->call_control_lock);
+               return -EBUSY;
+       }
+       ctx->call_control = control;
+       mutex_unlock(&ctx->call_control_lock);
+       if (!damon_is_running(ctx))
+               return -EINVAL;
+       wait_for_completion(&control->completion);
+       if (control->canceled)
+               return -ECANCELED;
+       return 0;
+}
+
 /*
  * Reset the aggregated monitoring results ('nr_accesses' of each region).
  */
@@ -1917,6 +1966,39 @@ static void kdamond_usleep(unsigned long usecs)
                usleep_range_idle(usecs, usecs + 1);
 }
 
+/*
+ * kdamond_call() - handle damon_call_control.
+ * @ctx:       The &struct damon_ctx of the kdamond.
+ * @cancel:    Whether to cancel the invocation of the function.
+ *
+ * If there is a &struct damon_call_control request that registered via
+ * &damon_call() on @ctx, do or cancel the invocation of the function depending
+ * on @cancel.  @cancel is set when the kdamond is deactivated by DAMOS
+ * watermarks, or the kdamond is already out of the main loop and therefore
+ * will be terminated.
+ */
+static void kdamond_call(struct damon_ctx *ctx, bool cancel)
+{
+       struct damon_call_control *control;
+       int ret = 0;
+
+       mutex_lock(&ctx->call_control_lock);
+       control = ctx->call_control;
+       mutex_unlock(&ctx->call_control_lock);
+       if (!control)
+               return;
+       if (cancel) {
+               control->canceled = true;
+       } else {
+               ret = control->fn(control->data);
+               control->return_code = ret;
+       }
+       complete(&control->completion);
+       mutex_lock(&ctx->call_control_lock);
+       ctx->call_control = NULL;
+       mutex_unlock(&ctx->call_control_lock);
+}
+
 /* Returns negative error code if it's not activated but should return */
 static int kdamond_wait_activation(struct damon_ctx *ctx)
 {
@@ -1941,6 +2023,7 @@ static int kdamond_wait_activation(struct damon_ctx *ctx)
                if (ctx->callback.after_wmarks_check &&
                                ctx->callback.after_wmarks_check(ctx))
                        break;
+               kdamond_call(ctx, true);
        }
        return -EBUSY;
 }
@@ -2011,6 +2094,7 @@ static int kdamond_fn(void *data)
                if (ctx->callback.after_sampling &&
                                ctx->callback.after_sampling(ctx))
                        break;
+               kdamond_call(ctx, false);
 
                kdamond_usleep(sample_interval);
                ctx->passed_sample_intervals++;
@@ -2072,6 +2156,8 @@ done:
        ctx->kdamond = NULL;
        mutex_unlock(&ctx->kdamond_lock);
 
+       kdamond_call(ctx, true);
+
        mutex_lock(&damon_lock);
        nr_running_ctxs--;
        if (!nr_running_ctxs && running_exclusive_ctxs)