bpf: Add generic support for lookup batch op

This commit introduces generic support for the bpf_map_lookup_batch.
This implementation can be used by almost all the bpf maps since its core
implementation is relying on the existing map_get_next_key and
map_lookup_elem. The bpf syscall subcommand introduced is:

  BPF_MAP_LOOKUP_BATCH

The UAPI attribute is:

  struct { /* struct used by BPF_MAP_*_BATCH commands */
         __aligned_u64   in_batch;       /* start batch,
                                          * NULL to start from beginning
                                          */
         __aligned_u64   out_batch;      /* output: next start batch */
         __aligned_u64   keys;
         __aligned_u64   values;
         __u32           count;          /* input/output:
                                          * input: # of key/value
                                          * elements
                                          * output: # of filled elements
                                          */
         __u32           map_fd;
         __u64           elem_flags;
         __u64           flags;
  } batch;

in_batch/out_batch are opaque values use to communicate between
user/kernel space, in_batch/out_batch must be of key_size length.

To start iterating from the beginning in_batch must be null,
count is the # of key/value elements to retrieve. Note that the 'keys'
buffer must be a buffer of key_size * count size and the 'values' buffer
must be value_size * count, where value_size must be aligned to 8 bytes
by userspace if it's dealing with percpu maps. 'count' will contain the
number of keys/values successfully retrieved. Note that 'count' is an
input/output variable and it can contain a lower value after a call.

If there's no more entries to retrieve, ENOENT will be returned. If error
is ENOENT, count might be > 0 in case it copied some values but there were
no more entries to retrieve.

Note that if the return code is an error and not -EFAULT,
count indicates the number of elements successfully processed.

Suggested-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Brian Vazquez <brianvv@google.com>
Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20200115184308.162644-3-brianvv@google.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index aed2bc39d72b6e47441a6806341b968707f34613..807744ecaa5a1a75542a32ffcde00bde971132eb 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -44,6 +44,8 @@ struct bpf_map_ops {
        int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
        void (*map_release_uref)(struct bpf_map *map);
        void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
+       int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
+                               union bpf_attr __user *uattr);
 
        /* funcs callable from userspace and from eBPF programs */
        void *(*map_lookup_elem)(struct bpf_map *map, void *key);
@@ -982,6 +984,9 @@ void *bpf_map_area_alloc(u64 size, int numa_node);
 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
 void bpf_map_area_free(void *base);
 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
+int  generic_map_lookup_batch(struct bpf_map *map,
+                             const union bpf_attr *attr,
+                             union bpf_attr __user *uattr);
 
 extern int sysctl_unprivileged_bpf_disabled;
 

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index a88837d4dd18ed4ab5ec183581867a383895fa9e..0721b2c6bb8c554e5feebcb1b3d7e464b3799eef 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -107,6 +107,7 @@ enum bpf_cmd {
        BPF_MAP_LOOKUP_AND_DELETE_ELEM,
        BPF_MAP_FREEZE,
        BPF_BTF_GET_NEXT_ID,
+       BPF_MAP_LOOKUP_BATCH,
 };
 
 enum bpf_map_type {
@@ -420,6 +421,23 @@ union bpf_attr {
                __u64           flags;
        };
 
+       struct { /* struct used by BPF_MAP_*_BATCH commands */
+               __aligned_u64   in_batch;       /* start batch,
+                                                * NULL to start from beginning
+                                                */
+               __aligned_u64   out_batch;      /* output: next start batch */
+               __aligned_u64   keys;
+               __aligned_u64   values;
+               __u32           count;          /* input/output:
+                                                * input: # of key/value
+                                                * elements
+                                                * output: # of filled elements
+                                                */
+               __u32           map_fd;
+               __u64           elem_flags;
+               __u64           flags;
+       } batch;
+
        struct { /* anonymous struct used by BPF_PROG_LOAD command */
                __u32           prog_type;      /* one of enum bpf_prog_type */
                __u32           insn_cnt;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 08b0b6e40454b15f82cd3c3203112c3ee5d41ffa..d604ddbb1afbe18af2c3863f2283520c789b0723 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -219,10 +219,8 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
        void *ptr;
        int err;
 
-       if (bpf_map_is_dev_bound(map)) {
-               err =  bpf_map_offload_lookup_elem(map, key, value);
-               return err;
-       }
+       if (bpf_map_is_dev_bound(map))
+               return bpf_map_offload_lookup_elem(map, key, value);
 
        preempt_disable();
        this_cpu_inc(bpf_prog_active);
@@ -1220,6 +1218,109 @@ err_put:
        return err;
 }
 
+#define MAP_LOOKUP_RETRIES 3
+
+int generic_map_lookup_batch(struct bpf_map *map,
+                                   const union bpf_attr *attr,
+                                   union bpf_attr __user *uattr)
+{
+       void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
+       void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
+       void __user *values = u64_to_user_ptr(attr->batch.values);
+       void __user *keys = u64_to_user_ptr(attr->batch.keys);
+       void *buf, *buf_prevkey, *prev_key, *key, *value;
+       int err, retry = MAP_LOOKUP_RETRIES;
+       u32 value_size, cp, max_count;
+       bool first_key = false;
+
+       if (attr->batch.elem_flags & ~BPF_F_LOCK)
+               return -EINVAL;
+
+       if ((attr->batch.elem_flags & BPF_F_LOCK) &&
+           !map_value_has_spin_lock(map))
+               return -EINVAL;
+
+       value_size = bpf_map_value_size(map);
+
+       max_count = attr->batch.count;
+       if (!max_count)
+               return 0;
+
+       if (put_user(0, &uattr->batch.count))
+               return -EFAULT;
+
+       buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
+       if (!buf_prevkey)
+               return -ENOMEM;
+
+       buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
+       if (!buf) {
+               kvfree(buf_prevkey);
+               return -ENOMEM;
+       }
+
+       err = -EFAULT;
+       first_key = false;
+       prev_key = NULL;
+       if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
+               goto free_buf;
+       key = buf;
+       value = key + map->key_size;
+       if (ubatch)
+               prev_key = buf_prevkey;
+
+       for (cp = 0; cp < max_count;) {
+               rcu_read_lock();
+               err = map->ops->map_get_next_key(map, prev_key, key);
+               rcu_read_unlock();
+               if (err)
+                       break;
+               err = bpf_map_copy_value(map, key, value,
+                                        attr->batch.elem_flags);
+
+               if (err == -ENOENT) {
+                       if (retry) {
+                               retry--;
+                               continue;
+                       }
+                       err = -EINTR;
+                       break;
+               }
+
+               if (err)
+                       goto free_buf;
+
+               if (copy_to_user(keys + cp * map->key_size, key,
+                                map->key_size)) {
+                       err = -EFAULT;
+                       goto free_buf;
+               }
+               if (copy_to_user(values + cp * value_size, value, value_size)) {
+                       err = -EFAULT;
+                       goto free_buf;
+               }
+
+               if (!prev_key)
+                       prev_key = buf_prevkey;
+
+               swap(prev_key, key);
+               retry = MAP_LOOKUP_RETRIES;
+               cp++;
+       }
+
+       if (err == -EFAULT)
+               goto free_buf;
+
+       if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
+                   (cp && copy_to_user(uobatch, prev_key, map->key_size))))
+               err = -EFAULT;
+
+free_buf:
+       kfree(buf_prevkey);
+       kfree(buf);
+       return err;
+}
+
 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
 
 static int map_lookup_and_delete_elem(union bpf_attr *attr)
@@ -3076,6 +3177,54 @@ out:
        return err;
 }
 
+#define BPF_MAP_BATCH_LAST_FIELD batch.flags
+
+#define BPF_DO_BATCH(fn)                       \
+       do {                                    \
+               if (!fn) {                      \
+                       err = -ENOTSUPP;        \
+                       goto err_put;           \
+               }                               \
+               err = fn(map, attr, uattr);     \
+       } while (0)
+
+static int bpf_map_do_batch(const union bpf_attr *attr,
+                           union bpf_attr __user *uattr,
+                           int cmd)
+{
+       struct bpf_map *map;
+       int err, ufd;
+       struct fd f;
+
+       if (CHECK_ATTR(BPF_MAP_BATCH))
+               return -EINVAL;
+
+       ufd = attr->batch.map_fd;
+       f = fdget(ufd);
+       map = __bpf_map_get(f);
+       if (IS_ERR(map))
+               return PTR_ERR(map);
+
+       if (cmd == BPF_MAP_LOOKUP_BATCH &&
+           !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
+               err = -EPERM;
+               goto err_put;
+       }
+
+       if (cmd != BPF_MAP_LOOKUP_BATCH &&
+           !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
+               err = -EPERM;
+               goto err_put;
+       }
+
+       if (cmd == BPF_MAP_LOOKUP_BATCH)
+               BPF_DO_BATCH(map->ops->map_lookup_batch);
+
+err_put:
+       fdput(f);
+       return err;
+}
+
 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
 {
        union bpf_attr attr = {};
@@ -3173,6 +3322,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
        case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
                err = map_lookup_and_delete_elem(&attr);
                break;
+       case BPF_MAP_LOOKUP_BATCH:
+               err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH);
+               break;
        default:
                err = -EINVAL;
                break;