bpf: libbpf: Add btf_line_info support to libbpf

This patch adds bpf_line_info support to libbpf:
1) Parsing the line_info sec from ".BTF.ext"
2) Relocating the line_info.  If the main prog *_info relocation
   fails, it will ignore the remaining subprog line_info and continue.
   If the subprog *_info relocation fails, it will bail out.
3) BPF_PROG_LOAD a prog with line_info

Signed-off-by: Martin KaFai Lau <kafai@fb.com>
Acked-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index 9fbbc0ed59525057960eee30bb97d81a4a9a9b5b..3caaa3428774427e451b50a682845b5ceb8eba18 100644 (file)
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
                                          -1);
 }
 
+static void *
+alloc_zero_tailing_info(const void *orecord, __u32 cnt,
+                       __u32 actual_rec_size, __u32 expected_rec_size)
+{
+       __u64 info_len = actual_rec_size * cnt;
+       void *info, *nrecord;
+       int i;
+
+       info = malloc(info_len);
+       if (!info)
+               return NULL;
+
+       /* zero out bytes kernel does not understand */
+       nrecord = info;
+       for (i = 0; i < cnt; i++) {
+               memcpy(nrecord, orecord, expected_rec_size);
+               memset(nrecord + expected_rec_size, 0,
+                      actual_rec_size - expected_rec_size);
+               orecord += actual_rec_size;
+               nrecord += actual_rec_size;
+       }
+
+       return info;
+}
+
 int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
                           char *log_buf, size_t log_buf_sz)
 {
+       void *finfo = NULL, *linfo = NULL;
        union bpf_attr attr;
-       void *finfo = NULL;
        __u32 name_len;
        int fd;
 
@@ -201,6 +226,9 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
        attr.func_info_rec_size = load_attr->func_info_rec_size;
        attr.func_info_cnt = load_attr->func_info_cnt;
        attr.func_info = ptr_to_u64(load_attr->func_info);
+       attr.line_info_rec_size = load_attr->line_info_rec_size;
+       attr.line_info_cnt = load_attr->line_info_cnt;
+       attr.line_info = ptr_to_u64(load_attr->line_info);
        memcpy(attr.prog_name, load_attr->name,
               min(name_len, BPF_OBJ_NAME_LEN - 1));
 
@@ -212,36 +240,35 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
         * to give user space a hint how to deal with loading failure.
         * Check to see whether we can make some changes and load again.
         */
-       if (errno == E2BIG && attr.func_info_cnt &&
-           attr.func_info_rec_size < load_attr->func_info_rec_size) {
-               __u32 actual_rec_size = load_attr->func_info_rec_size;
-               __u32 expected_rec_size = attr.func_info_rec_size;
-               __u32 finfo_cnt = load_attr->func_info_cnt;
-               __u64 finfo_len = actual_rec_size * finfo_cnt;
-               const void *orecord;
-               void *nrecord;
-               int i;
-
-               finfo = malloc(finfo_len);
-               if (!finfo)
-                       /* further try with log buffer won't help */
-                       return fd;
-
-               /* zero out bytes kernel does not understand */
-               orecord = load_attr->func_info;
-               nrecord = finfo;
-               for (i = 0; i < load_attr->func_info_cnt; i++) {
-                       memcpy(nrecord, orecord, expected_rec_size);
-                       memset(nrecord + expected_rec_size, 0,
-                              actual_rec_size - expected_rec_size);
-                       orecord += actual_rec_size;
-                       nrecord += actual_rec_size;
+       while (errno == E2BIG && (!finfo || !linfo)) {
+               if (!finfo && attr.func_info_cnt &&
+                   attr.func_info_rec_size < load_attr->func_info_rec_size) {
+                       /* try with corrected func info records */
+                       finfo = alloc_zero_tailing_info(load_attr->func_info,
+                                                       load_attr->func_info_cnt,
+                                                       load_attr->func_info_rec_size,
+                                                       attr.func_info_rec_size);
+                       if (!finfo)
+                               goto done;
+
+                       attr.func_info = ptr_to_u64(finfo);
+                       attr.func_info_rec_size = load_attr->func_info_rec_size;
+               } else if (!linfo && attr.line_info_cnt &&
+                          attr.line_info_rec_size <
+                          load_attr->line_info_rec_size) {
+                       linfo = alloc_zero_tailing_info(load_attr->line_info,
+                                                       load_attr->line_info_cnt,
+                                                       load_attr->line_info_rec_size,
+                                                       attr.line_info_rec_size);
+                       if (!linfo)
+                               goto done;
+
+                       attr.line_info = ptr_to_u64(linfo);
+                       attr.line_info_rec_size = load_attr->line_info_rec_size;
+               } else {
+                       break;
                }
 
-               /* try with corrected func info records */
-               attr.func_info = ptr_to_u64(finfo);
-               attr.func_info_rec_size = load_attr->func_info_rec_size;
-
                fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
 
                if (fd >= 0)
@@ -259,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
        fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
 done:
        free(finfo);
+       free(linfo);
        return fd;
 }
 

diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 098e6f793b7653ece3a98c959944d261d768d6e0..8f09de482839e289ff1b203d105eae26b52d2de6 100644 (file)
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -82,6 +82,9 @@ struct bpf_load_program_attr {
        __u32 func_info_rec_size;
        const void *func_info;
        __u32 func_info_cnt;
+       __u32 line_info_rec_size;
+       const void *line_info;
+       __u32 line_info_cnt;
 };
 
 /* Flags to direct loading requirements */

diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index aa4fa02b13fcb3a9e676cbd2664cd54cb61fb7c3..d682d3b8f7b9bdca25063bb23e49366ed5a54b42 100644 (file)
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -37,16 +37,26 @@ struct btf {
        int fd;
 };
 
+struct btf_ext_info {
+       /*
+        * info points to a deep copy of the individual info section
+        * (e.g. func_info and line_info) from the .BTF.ext.
+        * It does not include the __u32 rec_size.
+        */
+       void *info;
+       __u32 rec_size;
+       __u32 len;
+};
+
 struct btf_ext {
-       void *func_info;
-       __u32 func_info_rec_size;
-       __u32 func_info_len;
+       struct btf_ext_info func_info;
+       struct btf_ext_info line_info;
 };
 
-struct btf_sec_func_info {
+struct btf_ext_info_sec {
        __u32   sec_name_off;
-       __u32   num_func_info;
-       /* Followed by num_func_info number of bpf func_info records */
+       __u32   num_info;
+       /* Followed by num_info * record_size number of bytes */
        __u8    data[0];
 };
 
@@ -56,6 +66,14 @@ struct bpf_func_info_min {
        __u32   type_id;
 };
 
+/* The minimum bpf_line_info checked by the loader */
+struct bpf_line_info_min {
+       __u32   insn_off;
+       __u32   file_name_off;
+       __u32   line_off;
+       __u32   line_col;
+};
+
 static inline __u64 ptr_to_u64(const void *ptr)
 {
        return (__u64) (unsigned long) ptr;
@@ -486,12 +504,22 @@ exit_free:
        return err;
 }
 
-static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
-                                 __u8 *data, __u32 data_size,
-                                 btf_print_fn_t err_log)
+struct btf_ext_sec_copy_param {
+       __u32 off;
+       __u32 len;
+       __u32 min_rec_size;
+       struct btf_ext_info *ext_info;
+       const char *desc;
+};
+
+static int btf_ext_copy_info(struct btf_ext *btf_ext,
+                            __u8 *data, __u32 data_size,
+                            struct btf_ext_sec_copy_param *ext_sec,
+                            btf_print_fn_t err_log)
 {
        const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
-       const struct btf_sec_func_info *sinfo;
+       const struct btf_ext_info_sec *sinfo;
+       struct btf_ext_info *ext_info;
        __u32 info_left, record_size;
        /* The start of the info sec (including the __u32 record_size). */
        const void *info;
@@ -500,66 +528,69 @@ static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
        data = data + hdr->hdr_len;
        data_size -= hdr->hdr_len;
 
-       if (hdr->func_info_off & 0x03) {
-               elog("BTF.ext func_info section is not aligned to 4 bytes\n");
+       if (ext_sec->off & 0x03) {
+               elog(".BTF.ext %s section is not aligned to 4 bytes\n",
+                    ext_sec->desc);
                return -EINVAL;
        }
 
-       if (data_size < hdr->func_info_off ||
-           hdr->func_info_len > data_size - hdr->func_info_off) {
-               elog("func_info section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
-                    hdr->func_info_off, hdr->func_info_len);
+       if (data_size < ext_sec->off ||
+           ext_sec->len > data_size - ext_sec->off) {
+               elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
+                    ext_sec->desc, ext_sec->off, ext_sec->len);
                return -EINVAL;
        }
 
-       info = data + hdr->func_info_off;
-       info_left = hdr->func_info_len;
+       info = data + ext_sec->off;
+       info_left = ext_sec->len;
 
-       /* At least a func_info record size */
+       /* At least a record size */
        if (info_left < sizeof(__u32)) {
-               elog("BTF.ext func_info record size not found");
+               elog(".BTF.ext %s record size not found\n", ext_sec->desc);
                return -EINVAL;
        }
 
        /* The record size needs to meet the minimum standard */
        record_size = *(__u32 *)info;
-       if (record_size < sizeof(struct bpf_func_info_min) ||
+       if (record_size < ext_sec->min_rec_size ||
            record_size & 0x03) {
-               elog("BTF.ext func_info invalid record size");
+               elog("%s section in .BTF.ext has invalid record size %u\n",
+                    ext_sec->desc, record_size);
                return -EINVAL;
        }
 
        sinfo = info + sizeof(__u32);
        info_left -= sizeof(__u32);
 
-       /* If no func_info records, return failure now so .BTF.ext
-        * won't be used.
-        */
+       /* If no records, return failure now so .BTF.ext won't be used. */
        if (!info_left) {
-               elog("BTF.ext no func info records");
+               elog("%s section in .BTF.ext has no records", ext_sec->desc);
                return -EINVAL;
        }
 
        while (info_left) {
-               unsigned int sec_hdrlen = sizeof(struct btf_sec_func_info);
+               unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
                __u64 total_record_size;
                __u32 num_records;
 
                if (info_left < sec_hdrlen) {
-                       elog("BTF.ext func_info header not found");
+                       elog("%s section header is not found in .BTF.ext\n",
+                            ext_sec->desc);
                        return -EINVAL;
                }
 
-               num_records = sinfo->num_func_info;
+               num_records = sinfo->num_info;
                if (num_records == 0) {
-                       elog("incorrect BTF.ext num_func_info");
+                       elog("%s section has incorrect num_records in .BTF.ext\n",
+                            ext_sec->desc);
                        return -EINVAL;
                }
 
                total_record_size = sec_hdrlen +
                                    (__u64)num_records * record_size;
                if (info_left < total_record_size) {
-                       elog("incorrect BTF.ext num_func_info");
+                       elog("%s section has incorrect num_records in .BTF.ext\n",
+                            ext_sec->desc);
                        return -EINVAL;
                }
 
@@ -567,17 +598,49 @@ static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
                sinfo = (void *)sinfo + total_record_size;
        }
 
-       btf_ext->func_info_len = hdr->func_info_len - sizeof(__u32);
-       btf_ext->func_info_rec_size = record_size;
-       btf_ext->func_info = malloc(btf_ext->func_info_len);
-       if (!btf_ext->func_info)
+       ext_info = ext_sec->ext_info;
+       ext_info->len = ext_sec->len - sizeof(__u32);
+       ext_info->rec_size = record_size;
+       ext_info->info = malloc(ext_info->len);
+       if (!ext_info->info)
                return -ENOMEM;
-       memcpy(btf_ext->func_info, info + sizeof(__u32),
-              btf_ext->func_info_len);
+       memcpy(ext_info->info, info + sizeof(__u32), ext_info->len);
 
        return 0;
 }
 
+static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
+                                 __u8 *data, __u32 data_size,
+                                 btf_print_fn_t err_log)
+{
+       const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+       struct btf_ext_sec_copy_param param = {
+               .off = hdr->func_info_off,
+               .len = hdr->func_info_len,
+               .min_rec_size = sizeof(struct bpf_func_info_min),
+               .ext_info = &btf_ext->func_info,
+               .desc = "func_info"
+       };
+
+       return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
+}
+
+static int btf_ext_copy_line_info(struct btf_ext *btf_ext,
+                                 __u8 *data, __u32 data_size,
+                                 btf_print_fn_t err_log)
+{
+       const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+       struct btf_ext_sec_copy_param param = {
+               .off = hdr->line_info_off,
+               .len = hdr->line_info_len,
+               .min_rec_size = sizeof(struct bpf_line_info_min),
+               .ext_info = &btf_ext->line_info,
+               .desc = "line_info",
+       };
+
+       return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
+}
+
 static int btf_ext_parse_hdr(__u8 *data, __u32 data_size,
                             btf_print_fn_t err_log)
 {
@@ -617,7 +680,8 @@ void btf_ext__free(struct btf_ext *btf_ext)
        if (!btf_ext)
                return;
 
-       free(btf_ext->func_info);
+       free(btf_ext->func_info.info);
+       free(btf_ext->line_info.info);
        free(btf_ext);
 }
 
@@ -640,25 +704,32 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log)
                return ERR_PTR(err);
        }
 
+       err = btf_ext_copy_line_info(btf_ext, data, size, err_log);
+       if (err) {
+               btf_ext__free(btf_ext);
+               return ERR_PTR(err);
+       }
+
        return btf_ext;
 }
 
-int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext,
-                            const char *sec_name, __u32 insns_cnt,
-                            void **func_info, __u32 *cnt)
+static int btf_ext_reloc_info(const struct btf *btf,
+                             const struct btf_ext_info *ext_info,
+                             const char *sec_name, __u32 insns_cnt,
+                             void **info, __u32 *cnt)
 {
-       __u32 sec_hdrlen = sizeof(struct btf_sec_func_info);
-       __u32 i, record_size, existing_flen, records_len;
-       struct btf_sec_func_info *sinfo;
+       __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
+       __u32 i, record_size, existing_len, records_len;
+       struct btf_ext_info_sec *sinfo;
        const char *info_sec_name;
        __u64 remain_len;
        void *data;
 
-       record_size = btf_ext->func_info_rec_size;
-       sinfo = btf_ext->func_info;
-       remain_len = btf_ext->func_info_len;
+       record_size = ext_info->rec_size;
+       sinfo = ext_info->info;
+       remain_len = ext_info->len;
        while (remain_len > 0) {
-               records_len = sinfo->num_func_info * record_size;
+               records_len = sinfo->num_info * record_size;
                info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
                if (strcmp(info_sec_name, sec_name)) {
                        remain_len -= sec_hdrlen + records_len;
@@ -666,32 +737,52 @@ int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext,
                        continue;
                }
 
-               existing_flen = (*cnt) * record_size;
-               data = realloc(*func_info, existing_flen + records_len);
+               existing_len = (*cnt) * record_size;
+               data = realloc(*info, existing_len + records_len);
                if (!data)
                        return -ENOMEM;
 
-               memcpy(data + existing_flen, sinfo->data, records_len);
+               memcpy(data + existing_len, sinfo->data, records_len);
                /* adjust insn_off only, the rest data will be passed
                 * to the kernel.
                 */
-               for (i = 0; i < sinfo->num_func_info; i++) {
-                       struct bpf_func_info_min *record;
+               for (i = 0; i < sinfo->num_info; i++) {
+                       __u32 *insn_off;
 
-                       record = data + existing_flen + i * record_size;
-                       record->insn_off =
-                               record->insn_off / sizeof(struct bpf_insn) +
+                       insn_off = data + existing_len + (i * record_size);
+                       *insn_off = *insn_off / sizeof(struct bpf_insn) +
                                insns_cnt;
                }
-               *func_info = data;
-               *cnt += sinfo->num_func_info;
+               *info = data;
+               *cnt += sinfo->num_info;
                return 0;
        }
 
        return -ENOENT;
 }
 
+int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext,
+                            const char *sec_name, __u32 insns_cnt,
+                            void **func_info, __u32 *cnt)
+{
+       return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
+                                 insns_cnt, func_info, cnt);
+}
+
+int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext,
+                            const char *sec_name, __u32 insns_cnt,
+                            void **line_info, __u32 *cnt)
+{
+       return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
+                                 insns_cnt, line_info, cnt);
+}
+
 __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
 {
-       return btf_ext->func_info_rec_size;
+       return btf_ext->func_info.rec_size;
+}
+
+__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
+{
+       return btf_ext->line_info.rec_size;
 }

diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 936177a538cd587de357d893db334b4fcd39df77..b0610dcdae6bd7697fd6ae36bc38d107a691e60a 100644 (file)
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -51,6 +51,8 @@ struct btf_ext_header {
        /* All offsets are in bytes relative to the end of this header */
        __u32   func_info_off;
        __u32   func_info_len;
+       __u32   line_info_off;
+       __u32   line_info_len;
 };
 
 typedef int (*btf_print_fn_t)(const char *, ...)
@@ -70,10 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
 
 struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log);
 void btf_ext__free(struct btf_ext *btf_ext);
-int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext,
+int btf_ext__reloc_func_info(const struct btf *btf,
+                            const struct btf_ext *btf_ext,
                             const char *sec_name, __u32 insns_cnt,
                             void **func_info, __u32 *func_info_len);
+int btf_ext__reloc_line_info(const struct btf *btf,
+                            const struct btf_ext *btf_ext,
+                            const char *sec_name, __u32 insns_cnt,
+                            void **line_info, __u32 *cnt);
 __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext);
+__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext);
 
 #ifdef __cplusplus
 } /* extern "C" */

diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 4ea3368bf8037acda8607f2c8799d6ea24f2535b..e2bc75ee16146e4446c4e13a010c6540d039293c 100644 (file)
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -170,6 +170,10 @@ struct bpf_program {
        __u32 func_info_cnt;
 
        struct bpf_capabilities *caps;
+
+       void *line_info;
+       __u32 line_info_rec_size;
+       __u32 line_info_cnt;
 };
 
 struct bpf_map {
@@ -1342,6 +1346,19 @@ bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
                prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
        }
 
+       if (!insn_offset || prog->line_info) {
+               err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
+                                              section_name, insn_offset,
+                                              &prog->line_info,
+                                              &prog->line_info_cnt);
+               if (err)
+                       return check_btf_ext_reloc_err(prog, err,
+                                                      prog->line_info,
+                                                      "bpf_line_info");
+
+               prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
+       }
+
        if (!insn_offset)
                prog->btf_fd = btf__fd(obj->btf);
 
@@ -1526,6 +1543,9 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
        load_attr.func_info = prog->func_info;
        load_attr.func_info_rec_size = prog->func_info_rec_size;
        load_attr.func_info_cnt = prog->func_info_cnt;
+       load_attr.line_info = prog->line_info;
+       load_attr.line_info_rec_size = prog->line_info_rec_size;
+       load_attr.line_info_cnt = prog->line_info_cnt;
        if (!load_attr.insns || !load_attr.insns_cnt)
                return -EINVAL;