ctf: avoid command-line length limits by passing .o filenames via a file

Historically, dwarf2ctf took the names of all object files it was to run over on
the command line.  This can lead to very long command lines, but since the kernel
now supports lines of almost unlimited length (up to RLIMIT_STACK / 4, which is
much longer than the names of all .c files in the kernel tree added together for
any reasonable stack size), this seemed safe.

It turns out not to be, because GNU Make sometimes passes command lines through
the shell even when it doesn't need to, and GNU Bash does not yet know that the
kernel has a very long command-line length limit and protests if it is
> ARG_MAX, which is much shorter.  We were trying to prevent the dwarf2ctf
command line being passed to the shell, but there is no documented way to do so
in GNU Make and clearly simply avoiding shell metacharacters is not always
enough.

So, instead, write out the list of object files to a file in the .ctf directory
and have dwarf2ctf read it in.  This takes some trickery in Makefile.modpost to
arrange to invoke a separate shell to add each filename to the filelist, but
it's not too complicated.  (To minimize the possibility of disruption, we
duplicate the code already used for reading blacklists in dwarf2ctf.  This
should be unified later.)

Orabug: 17363469
Signed-off-by: Nick Alcock <nick.alcock@oracle.com>

diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost
index 322a332de32d8a1bebf541953c41d2a5eca6269a..cd6001ad57d47fe9ca44c05af98400d289100ee6 100644 (file)
--- a/scripts/Makefile.modpost
+++ b/scripts/Makefile.modpost
@@ -139,17 +139,16 @@ ifndef CONFIG_DT_DISABLE_CTF
 # Out-of-tree module CTF gets its own per-module set of stamp files, since its
 # CTF is rebuilt independently.
 #
-# Warning: cmd_ctf can expand to an emormously long command line, long enough
-# that many shells dump core trying to parse it.  We must avoid most shell
-# metacharacters in the definition of cmd_ctf, whereupon GNU make will invoke
-# the command directly rather than going through $SHELL.
+# We write the names of the object files to be scanned for CTF content into a
+# file, then use that, to avoid hitting command-line length limits.
 
 ifeq ($(KBUILD_EXTMOD),)
 ctf-dir := .ctf
 quiet_cmd_ctf = CTF
-      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf $(ctf-dir) objects.builtin modules.builtin scripts/dwarf2ctf/dedup.blacklist scripts/dwarf2ctf/member.blacklist $^
+      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf $(ctf-dir) objects.builtin modules.builtin scripts/dwarf2ctf/dedup.blacklist scripts/dwarf2ctf/member.blacklist $(ctf-filelist)
 builtins := $(shell cat objects.builtin 2>/dev/null)
 ctf-stamp := .ctf/ctf.stamp
+ctf-filelist := .ctf/ctf.filelist
 
 # The CTF module depends on the CTF stamp file, in lieu of the builtin
 # CTF files whose names we cannot determine until it is too late.
@@ -159,9 +158,10 @@ kernel/ctf/ctf.ko: .ctf/ctf.stamp
 else
 ctf-dir := $(KBUILD_EXTMOD)/.ctf
 quiet_cmd_ctf = CTF
-      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf $(ctf-dir) -e $^
+      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf $(ctf-dir) -e $(ctf-filelist)
 builtins :=
 ctf-stamp := $(ctf-dir)/$(notdir $(M)-extmod).stamp
+ctf-filelist := $(ctf-dir)/$(notdir $(M)-extmod).ctf.filelist
 
 endif
 
@@ -170,10 +170,26 @@ endif
 all-module-ctfs = $(addprefix $(ctf-dir)/,$(notdir $(modules:.ko=.mod.ctf)))
 $(all-module-ctfs): $(ctf-stamp)
 
+# This ensures that the (possibly very long) prerequisite list is written out
+# using one shell invocation per prerequisite.
+#
+# The empty line in this definition is crucial.
+define add-ctf-filelist-cmd
+       @echo "$(1)" >> $(ctf-filelist)
+
+endef
+define add-ctf-filelists-cmd
+       $(foreach file,$(1),$(call add-ctf-filelist-cmd,$(file)))
+endef
+
+$(ctf-filelist): $(builtins) $(modules:.ko=.o)
+       @rm -f $(ctf-filelist);
+       $(call add-ctf-filelists-cmd,$^)
+
 # We depend upon a stamp file in lieu of the builtin modules' CTF files, because
 # the names of the generated CTF files for the builtins are so variable.
 # (Standalone modules get their own per-module stamp files.)
-$(ctf-stamp): $(builtins) $(modules:.ko=.o)
+$(ctf-stamp): $(ctf-filelist)
        $(call if_changed,ctf)
        @shopt -s nullglob; \
        for name in $(ctf-dir)/*.builtin.ctf; do \

diff --git a/scripts/dwarf2ctf/dwarf2ctf.c b/scripts/dwarf2ctf/dwarf2ctf.c
index b0ad0f75a63f571bef4138cefce22255d4ec7acb..7e75d24ae20992d8fb4e035a539de288274f64c6 100644 (file)
--- a/scripts/dwarf2ctf/dwarf2ctf.c
+++ b/scripts/dwarf2ctf/dwarf2ctf.c
@@ -54,10 +54,9 @@ static const char *trace;
 /*
  * Run dwarf2ctf over a single object file or set thereof.
  *
- * starting_argv is the point in the argv array at which the arguments start.
  * output_dir is the directory into which the CTF goes.
  */
-static void run(int starting_argv, char *argv[], char *output_dir);
+static void run(char *output_dir);
 
 /*
  * A fully descriptive CTF type ID: both file and type ID in one place.
@@ -104,6 +103,18 @@ static GHashTable *id_to_module;
  */
 static GHashTable *module_to_ctf_file;
 
+/*
+ * The names of the object files to run over.  Except in -e mode, this comes
+ * straight from the module filelist passed in.
+ */
+static char **object_names;
+static size_t object_names_cnt;
+
+/*
+ * Populate the object_names list from the module filelist.
+ */
+static void init_object_names(const char *object_names_file);
+
 /*
  * The names of module object files presently built in to the kernel, in the
  * same format as the module names in tu_to_module.
@@ -254,7 +265,7 @@ static void process_tu_func(const char *module_name,
 /*
  * Scan and identify duplicates across the entire set of object files.
  */
-static void scan_duplicates(int starting_argv, char *argv[]);
+static void scan_duplicates(void);
 
 /*
  * Recursively detect duplicate types and types referenced by them, and
@@ -654,16 +665,15 @@ static void private_ctf_free(void *ctf_file);
 
 int main(int argc, char *argv[])
 {
-       int starting_argv = 3;
        char *output_dir;
 
        trace = getenv("DWARF2CTF_TRACE");
 
-       if (argc < 2 || strcmp(argv[1], "-e") == 0 ||
-           (argc < 6 && (strcmp(argv[2], "-e") != 0))) {
+       if ((argc != 4 && argc != 7) ||
+           (argc == 4 && strcmp(argv[2], "-e") != 0)) {
                fprintf(stderr, "Syntax: dwarf2ctf outputdir objects.builtin modules.builtin dedup.blacklist\n");
-               fprintf(stderr, "                  member.blacklist vmlinux.o module.o...,\n");
-               fprintf(stderr, "        or dwarf2ctf outputdir -e module.o ... "
+               fprintf(stderr, "                  member.blacklist filelist\n");
+               fprintf(stderr, "    or dwarf2ctf outputdir -e filelist"
                        "for external module use\n");
                exit(1);
        }
@@ -695,21 +705,34 @@ int main(int argc, char *argv[])
                builtin_module_file = argv[3];
                dedup_blacklist_file = argv[4];
                member_blacklist_file = argv[5];
-               starting_argv = 6;
+
                init_builtin(builtin_objects_file, builtin_module_file);
                init_dedup_blacklist(dedup_blacklist_file);
                init_member_blacklist(member_blacklist_file);
+               init_object_names(argv[6]);
 
-               run(starting_argv, argv, output_dir);
+               run(output_dir);
        } else {
-               char **name;
+               char *single_object_name;
+               char **all_object_names;
+               size_t all_object_names_cnt;
+               size_t i;
 
-               for (name = &argv[starting_argv]; *name; name++) {
-                       char *one_argv[2];
-                       one_argv[0] = *name;
-                       one_argv[1] = NULL;
+               init_object_names(argv[3]);
 
-                       run(0, one_argv, output_dir);
+               /*
+                * Repeatedly populate object_names with one object name, and
+                * call run() with that.
+                */
+               all_object_names = object_names;
+               all_object_names_cnt = object_names_cnt;
+               object_names = &single_object_name;
+               object_names_cnt = 1;
+
+               for (i = 0; i < all_object_names_cnt; i++) {
+                       single_object_name = all_object_names[i];
+
+                       run(output_dir);
                }
        }
 
@@ -722,12 +745,11 @@ int main(int argc, char *argv[])
 /*
  * Run dwarf2ctf over a single object file or set thereof.
  *
- * starting_argv is the point in the argv array at which the arguments start.
  * output_dir is the directory into which the CTF goes.
  */
-static void run(int starting_argv, char *argv[], char *output_dir)
+static void run(char *output_dir)
 {
-       char **name;
+       size_t i;
 
        /*
         * Create all the hashes, assemble the translation unit->module list for
@@ -749,14 +771,14 @@ static void run(int starting_argv, char *argv[], char *output_dir)
        if (builtin_modules != NULL)
                init_ctf_table("shared_ctf");
 
-       scan_duplicates(starting_argv, argv);
+       scan_duplicates();
 
        /*
         * Now construct CTF out of the types.
         */
        dw_ctf_trace("CTF construction.\n");
-       for (name = &argv[starting_argv]; *name; name++)
-               process_file(*name, construct_ctf, NULL, NULL, NULL);
+       for (i = 0; i < object_names_cnt; i++)
+               process_file(object_names[i], construct_ctf, NULL, NULL, NULL);
 
        /*
         * Finally, emit the types into their .ctf files, and generate the
@@ -771,6 +793,59 @@ static void run(int starting_argv, char *argv[], char *output_dir)
        g_hash_table_destroy(module_to_ctf_file);
 }
 
+
+/*
+ * Populate the builtin_modules and builtin_objects lists from the
+ * objects.builtin and modules.builtin file.
+ */
+static void init_object_names(const char *object_names_file)
+{
+       FILE *f;
+       char *line = NULL;
+       size_t line_size = 0;
+
+       if ((f = fopen(object_names_file, "r")) == NULL) {
+               fprintf(stderr, "Cannot open object names file %s: "
+                       "%s\n", object_names_file, strerror(errno));
+               exit(1);
+       }
+
+       /*
+        * This needs no massaging other than linefeed removal, just reading and
+        * stashing.
+        */
+
+       while (getline(&line, &line_size, f) >= 0) {
+               size_t len = strlen(line);
+
+               if (len == 0)
+                       continue;
+
+               if (line[len-1] == '\n')
+                       line[len-1] = '\0';
+
+               object_names = realloc(object_names,
+                                      ++object_names_cnt *
+                                      sizeof (char *));
+
+               if (object_names == NULL) {
+                       fprintf(stderr, "Out of memory reading %s",
+                               object_names_file);
+                       exit(1);
+               }
+
+               object_names[object_names_cnt-1] = xstrdup(line);
+       }
+
+       if (ferror(f)) {
+               fprintf(stderr, "Error reading from %s: %s\n",
+                       object_names_file, strerror(errno));
+               exit(1);
+       }
+
+       fclose(f);
+}
+
 /*
  * Populate the builtin_modules and builtin_objects lists from the
  * objects.builtin and modules.builtin file.
@@ -1607,9 +1682,9 @@ static void process_tu_func(const char *module_name,
 /*
  * Scan and identify duplicates across the entire set of object files.
  */
-static void scan_duplicates(int starting_argv, char *argv[])
+static void scan_duplicates(void)
 {
-       char **name;
+       size_t i;
 
        /*
         * First, determine which types are referenced by more than one
@@ -1630,8 +1705,8 @@ static void scan_duplicates(int starting_argv, char *argv[])
        dw_ctf_trace("Duplicate detection: primary pass.\n");
 
        state.repeat_detection = 0;
-       for (name = &argv[starting_argv]; *name; name++)
-               process_file(*name, detect_duplicates,
+       for (i = 0; i < object_names_cnt; i++)
+               process_file(object_names[i], detect_duplicates,
                             detect_duplicates_init,
                             detect_duplicates_done, &state);
 
@@ -1651,8 +1726,9 @@ static void scan_duplicates(int starting_argv, char *argv[])
 
                state.repeat_detection = 0;
 
-               for (name = &argv[starting_argv]; *name; name++)
-                       process_file(*name, detect_duplicates_alias_fixup,
+               for (i = 0; i < object_names_cnt; i++)
+                       process_file(object_names[i],
+                                    detect_duplicates_alias_fixup,
                                     detect_duplicates_init,
                                     detect_duplicates_done, &state);
        } while (state.repeat_detection);